dvc: bump to 2.2.0

astroid==2.5.7 has improved inference and was throwing
errors that were false negatives before. Fixing these
and ignoring some.

Also added `funcy.decorator` to the `signature-mutators`,
as it changes function signature (i.e. provides `Call` as
a first argument which is not required to pass).

Example errors that we were getting because of this:

```
tests/remotes/gdrive.py:40:11: E1120: No value for argument 'tries' in function call (no-value-for-parameter)
************* Module dvc.stage.serialize
dvc/stage/serialize.py:38:1: E1120: No value for argument 'func' in
function call (no-value-for-parameter)
************* Module dvc.fs.http
dvc/fs/http.py:19:1: E1120: No value for argument 'ctx' in function call (no-value-for-parameter)
************* Module dvc.fs.gdrive
dvc/fs/gdrive.py:66:11: E1120: No value for argument 'tries' in function call (no-value-for-parameter)
dvc/fs/gdrive.py:459:5: E1120: No value for argument 'ctx' in function call (no-value-for-parameter)
```

.coveragerc

@@ -13,3 +13,4 @@ exclude_lines =
     raise NotImplementedError
     # pragma: no cover
     #  pragma: no cover
+    \.\.\.$

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+
+updates:
+  - directory: "/"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"
+
+  - directory: "/"
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"

.github/workflows/packages.yaml

@@ -9,13 +9,13 @@ jobs:
   deb:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
+      uses: actions/setup-ruby@v1.1.3
       with:
         ruby-version: 2.4
     - name: Install deps
@@ -25,14 +25,9 @@ jobs:
         gem install --no-document fpm
     - name: Build deb
       run: python scripts/build.py deb
-    - name: Upload deb artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.deb
-        if-no-files-found: error
     - name: Publish deb
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -43,13 +38,13 @@ jobs:
   rpm:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
+      uses: actions/setup-ruby@v1.1.3
       with:
         ruby-version: 2.4
     - name: Install deps
@@ -59,14 +54,9 @@ jobs:
         gem install --no-document fpm
     - name: Build rpm
       run: python scripts/build.py rpm
-    - name: Upload rpm artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.rpm
-        if-no-files-found: error
     - name: Publish rpm
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -77,13 +67,13 @@ jobs:
   osxpkg:
     runs-on: macos-10.15
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
+      uses: actions/setup-ruby@v1.1.3
       with:
         ruby-version: 2.4
     - name: Install deps
@@ -96,14 +86,9 @@ jobs:
         gem install --no-document fpm
     - name: Build
       run: python scripts/build.py osxpkg
-    - name: Upload osxpkg artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.pkg
-        if-no-files-found: error
     - name: Publish
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -114,9 +99,9 @@ jobs:
   exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Install deps
@@ -129,16 +114,11 @@ jobs:
     - name: Build
       run: python scripts\build.py exe
       shell: cmd
-    - name: Upload exe artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/innosetup/dvc*.exe
-        if-no-files-found: error
     - name: Publish
       if: github.event_name == 'release'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         upload_url: ${{ github.event.release.upload_url }}
         asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
@@ -147,9 +127,9 @@ jobs:
   pip:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.7
     - name: Install
@@ -162,29 +142,3 @@ jobs:
       with:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
-  snap:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/[email protected]
-    - name: Build the package
-      uses: snapcore/action-build@v1
-      id: snapcraft
-#    - name: Test the package
-#      run: |
-#        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
-#        dvc version
-#        sudo snap remove dvc
-    - name: Publish to the edge channel
-      uses: snapcore/action-publish@v1
-      if: github.event_name == 'push'
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: edge,v2/edge
-    - name: Publish to the stable channel
-      uses: snapcore/action-publish@v1
-      if: ${{ github.event_name == 'release' && github.event.release.released }}
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: stable,v2/stable

.github/workflows/tests.yaml

@@ -7,34 +7,34 @@ on:
   push:
     branches:
       - master
+  schedule:
+    - cron: '5 1 * * *'  # every day at 01:05
 
 env:
   DVC_TEST: "true"
   HOMEBREW_NO_AUTO_UPDATE: 1
   SHELL: /bin/bash
-  GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Install requirements
-      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+      run: |
+        pip install wheel
+        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
         pip install git+https://github.com/isidentical/pydata-google-auth@service-account
     - name: Check README
       run: python setup.py checkdocs
-    - uses: pre-commit/[email protected].0
+    - uses: pre-commit/[email protected].3
   tests:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -51,9 +51,9 @@ jobs:
             pyv: "3.9"
 
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -62,7 +62,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v2.1.6
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
@@ -83,14 +83,44 @@ jobs:
         git config --global user.name "DVC Tester"
     - name: setup gs creds
       shell: bash
+      env:
+        GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
       if: env.GS_CREDS_JSON != ''
       run: |
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
+    - name: disable gdrive tests
+      if: >-
+        github.event_name != 'schedule' &&
+        (matrix.pyv != '3.8' || matrix.os != 'ubuntu-18.04')
+      run: |
+        echo "extra_test_args=--disable-gdrive" >> $GITHUB_ENV
     - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+      env:
+        GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      run: >-
+        python -m tests -n=4
+        --cov-report=xml --cov-report=term
+        --all ${{ env.extra_test_args }}
     - name: upload coverage report
-      uses: codecov/codecov-action@v1.0.13
+      uses: codecov/codecov-action@v1.5.0
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
+  notify:
+    if: github.ref == 'refs/heads/master' && failure()
+    needs:
+      - lint
+      - tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Slack Notification
+      uses: rtCamp/[email protected]
+      env:
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        SLACK_COLOR: ${{ job.status }}
+        SLACK_MESSAGE: 'CI Failed on master :boom:'
+        SLACK_TITLE: CI Status
+        SLACK_USERNAME: DVC-CI

.pre-commit-config.yaml

@@ -3,12 +3,12 @@ repos:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 19.10b0
+    rev: 21.5b1
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.8.0
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,13 +18,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.2
   - hooks:
       -  id: mypy
          language_version: python3
          files: ^dvc/
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
+    rev: 'v0.812'
   - repo: local
     hooks:
     - id: pylint
@@ -39,7 +39,7 @@ repos:
         id: beautysh
         language_version: python3
     repo: https://github.com/lovesegfault/beautysh
-    rev: 6.0.1
+    rev: v6.1.0
   - hooks:
       - args:
           - git-hook

.pylintrc

@@ -313,6 +313,7 @@ missing-member-hint-distance=1
 # showing a hint for a missing member.
 missing-member-max-choices=1
 
+signature-mutators=funcy.decorators.decorator,
 
 [VARIABLES]
 

MANIFEST.in

@@ -1,2 +1,3 @@
 include fastentrypoints.py
 include LICENSE
+include test_requirements.txt

README.rst

@@ -129,7 +129,11 @@ Conda (Anaconda)
 
 .. code-block:: bash
 
-   conda install -c conda-forge dvc
+   conda install -c conda-forge mamba # installs much faster than conda
+   mamba install -c conda-forge dvc
+
+Depending on the remote storage type you plan to use to keep and share your data, you might need to
+install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`.
 
 pip (PyPI)
 ----------
@@ -178,6 +182,12 @@ Fedora / CentOS (rpm)
 Comparison to related technologies
 ==================================
 
+#. *Data Engineering* tools such as `AirFlow <https://airflow.apache.org/>`_, `Luigi <https://github.com/spotify/luigi>`_ and
+   others - in DVC data, model and ML pipelines represent a single ML project and focuses on data scientist' experience while
+   data engineering pipeline orchestrates multiple data projects and focuses on efficent execution. DVC project can be used
+   from data pipeline as a single execution step. `AirFlow DVC <https://github.com/covid-genomics/airflow-dvc>`_ is an example
+   of such integration.
+
 #. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
    not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
    copying/duplicating files.
@@ -191,7 +201,8 @@ Comparison to related technologies
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
+#. `DAGsHub <https://dagshub.com/>`_ - online service of Git+DVC repositories with pipeline and metrics visualization and
+   DVC-specific cloud storage.
 
 Contributing
 ============

debian/changelog

@@ -1,5 +0,0 @@
-dvc (0.75.b1) experimental; urgency=low
-
-  * First native Debian package version.
-
- -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,33 +0,0 @@
-Source: dvc
-Maintainer: Dmitry Petrov <[email protected]>
-Section: python
-Priority: optional
-Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
-Standards-Version: 4.0.0
-Homepage: http://dvc.org
-Vcs-Git: https://github.com/iterative/dvc.git
-Vcs-browser: https://github.com/iterative/dvc
-
-Package: dvc
-Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
-Recommends: python3-arrow (=0.14.0),
-            python3-paramiko (>=2.5.0),
-            python3-gssapi (>= 2.5.0),
-            python3-boto3 (>=1.9.86),
-            python3-pydrive (=1.3.1),
-            python3-azure-storage-blob (=12.1.0),
-            python3-oss2 (=2.6.1),
-            python3-google-cloud-storage (=1.19.0)
-Description: Git for data scientists - manage your code and data together
- Data Version Control or DVC is an open-source tool for data science and machine learning projects.
- .
- Key features:
- .
- 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
- .
- 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
- .
- 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
- .
- 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -1,11 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <[email protected]>
-Source: https://github.com/iterative/dvc
-
-Files: *
-Copyright: Dmitry Petrov <[email protected]>
-License: Apache 2.0
- On Debian systems, the full text of the Apache 2.0
- License can be found in the file
- `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -1,6 +0,0 @@
-#!/usr/bin/make -f
-
-export PYBUILD_NAME=dvc
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/watch

@@ -1,4 +0,0 @@
-# please also check http://pypi.debian.net/dvc/watch
-version=3
-opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
-http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/_debug.py

@@ -77,15 +77,15 @@ def add_debugging_flags(parser):
     from argparse import SUPPRESS
 
     parser.add_argument(
-        "--cprofile", action="store_true", default=False, help=SUPPRESS,
+        "--cprofile", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument("--cprofile-dump", help=SUPPRESS)
     parser.add_argument(
-        "--pdb", action="store_true", default=False, help=SUPPRESS,
+        "--pdb", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument(
-        "--instrument", action="store_true", default=False, help=SUPPRESS,
+        "--instrument", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument(
-        "--instrument-open", action="store_true", default=False, help=SUPPRESS,
+        "--instrument-open", action="store_true", default=False, help=SUPPRESS
     )

dvc/checkout.py

@@ -2,19 +2,21 @@ import logging
 
 from shortuuid import uuid
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.exceptions import (
     CacheLinkError,
     CheckoutError,
     ConfirmRemoveError,
     DvcException,
 )
+from dvc.ignore import DvcIgnoreFilter
 from dvc.objects import check, load
 from dvc.objects.errors import ObjectFormatError
 from dvc.objects.stage import stage
 from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
     slow_link_guard,
 )
+from dvc.types import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -95,7 +97,7 @@ def _do_link(cache, from_info, to_info, link_method):
     link_method(from_info, to_info)
 
     logger.debug(
-        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info,
+        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info
     )
 
 
@@ -148,7 +150,14 @@ def _cache_is_copy(cache, path_info):
 
 
 def _checkout_file(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
+    path_info,
+    fs,
+    obj,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    state=None,
 ):
     """The file is changed we need to checkout a new copy"""
     modified = False
@@ -168,15 +177,22 @@ def _checkout_file(
         _link(cache, cache_info, path_info)
         modified = True
 
-    fs.repo.state.save(path_info, fs, obj.hash_info)
+    if state:
+        state.save(path_info, fs, obj.hash_info)
+
     if progress_callback:
         progress_callback(str(path_info))
 
     return modified
 
 
-def _remove_redundant_files(path_info, fs, obj, cache, force):
-    existing_files = set(fs.walk_files(path_info))
+def _remove_redundant_files(
+    path_info, fs, obj, cache, force, dvcignore: Optional[DvcIgnoreFilter]
+):
+    if dvcignore:
+        existing_files = set(dvcignore.walk_files(fs, path_info))
+    else:
+        existing_files = set(fs.walk_files(path_info))
 
     needed_files = {path_info.joinpath(*key) for key, _ in obj}
     redundant_files = existing_files - needed_files
@@ -187,9 +203,17 @@ def _remove_redundant_files(path_info, fs, obj, cache, force):
 
 
 def _checkout_dir(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
+    path_info,
+    fs,
+    obj,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
 ):
-    modified = False, False
+    modified = False
     # Create dir separately so that dir is created
     # even if there are no files in it
     if not fs.exists(path_info):
@@ -207,15 +231,20 @@ def _checkout_dir(
             force,
             progress_callback,
             relink,
+            state=None,
         )
         if entry_modified:
             modified = True
 
     modified = (
-        _remove_redundant_files(path_info, fs, obj, cache, force) or modified
+        _remove_redundant_files(
+            path_info, fs, obj, cache, force, dvcignore=dvcignore
+        )
+        or modified
     )
 
-    fs.repo.state.save(path_info, fs, obj.hash_info)
+    if state:
+        state.save(path_info, fs, obj.hash_info)
 
     # relink is not modified, assume it as nochange
     return modified and not relink
@@ -229,17 +258,28 @@ def _checkout(
     force=False,
     progress_callback=None,
     relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
 ):
     if not obj.hash_info.isdir:
         ret = _checkout_file(
-            path_info, fs, obj, cache, force, progress_callback, relink
+            path_info, fs, obj, cache, force, progress_callback, relink, state
         )
     else:
         ret = _checkout_dir(
-            path_info, fs, obj, cache, force, progress_callback, relink,
+            path_info,
+            fs,
+            obj,
+            cache,
+            force,
+            progress_callback,
+            relink,
+            dvcignore=dvcignore,
+            state=state,
         )
 
-    fs.repo.state.save_link(path_info, fs)
+    if state:
+        state.save_link(path_info, fs)
 
     return ret
 
@@ -253,6 +293,8 @@ def checkout(
     progress_callback=None,
     relink=False,
     quiet=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
 ):
     if path_info.scheme not in ["local", cache.fs.scheme]:
         raise NotImplementedError
@@ -269,7 +311,7 @@ def checkout(
         failed = path_info
 
     elif not relink and not _changed(path_info, fs, obj, cache):
-        logger.trace("Data '%s' didn't change.", path_info)
+        logger.trace("Data '%s' didn't change.", path_info)  # type: ignore
         skip = True
     else:
         try:
@@ -286,9 +328,7 @@ def checkout(
 
     if failed or skip:
         if progress_callback and obj:
-            progress_callback(
-                str(path_info), len(obj),
-            )
+            progress_callback(str(path_info), len(obj))
         if failed:
             raise CheckoutError([failed])
         return
@@ -296,5 +336,13 @@ def checkout(
     logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
 
     return _checkout(
-        path_info, fs, obj, cache, force, progress_callback, relink,
+        path_info,
+        fs,
+        obj,
+        cache,
+        force,
+        progress_callback,
+        relink,
+        dvcignore=dvcignore,
+        state=state,
     )

dvc/command/add.py

@@ -117,6 +117,6 @@ def add_parser(subparsers, parent_parser):
         ),
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add.",
+        "targets", nargs="+", help="Input files/directories to add."
     ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -7,11 +7,11 @@ logger = logging.getLogger(__name__)
 
 def fix_subparsers(subparsers):
     """Workaround for bug in Python 3. See more info at:
-        https://bugs.python.org/issue16308
-        https://github.com/iterative/dvc/issues/769
+    https://bugs.python.org/issue16308
+    https://github.com/iterative/dvc/issues/769
 
-        Args:
-            subparsers: subparsers to fix.
+    Args:
+        subparsers: subparsers to fix.
     """
     subparsers.required = True
     subparsers.dest = "cmd"
@@ -45,6 +45,10 @@ class CmdBase(ABC):
             updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
             updater.check()
 
+    def do_run(self):
+        with self.repo:
+            return self.run()
+
     @abstractmethod
     def run(self):
         pass
@@ -55,3 +59,6 @@ class CmdBaseNoRepo(CmdBase):
         self.args = args
 
         os.chdir(args.cd)
+
+    def do_run(self):
+        return self.run()

dvc/command/cache.py

@@ -1,11 +1,9 @@
 import argparse
-import logging
 
 from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 class CmdCacheDir(CmdConfig):
@@ -17,7 +15,7 @@ class CmdCacheDir(CmdConfig):
                 # Use merged config with default values
                 conf = self.config
             self._check(conf, False, "cache", "dir")
-            logger.info(conf["cache"]["dir"])
+            ui.write(conf["cache"]["dir"])
             return 0
         with self.config.edit(level=self.args.level) as conf:
             if self.args.unset:

dvc/command/check_ignore.py

@@ -1,16 +1,14 @@
 import argparse
-import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 class CmdCheckIgnore(CmdBase):
     def __init__(self, args):
         super().__init__(args)
-        self.ignore_filter = self.repo.fs.dvcignore
+        self.ignore_filter = self.repo.dvcignore
 
     def _show_results(self, result):
         if not result.match and not self.args.non_matching:
@@ -22,9 +20,9 @@ class CmdCheckIgnore(CmdBase):
                 patterns = patterns[-1:]
 
             for pattern in patterns:
-                logger.info("{}\t{}".format(pattern, result.file))
+                ui.write(pattern, result.file, sep="\t")
         else:
-            logger.info(result.file)
+            ui.write(result.file)
 
     def _check_one_file(self, target):
         result = self.ignore_filter.check_ignore(target)
@@ -122,6 +120,6 @@ def add_parser(subparsers, parent_parser):
         help="Read paths from standard input instead of providing `targets`.",
     )
     parser.add_argument(
-        "targets", nargs="*", help="File or directory paths to check",
+        "targets", nargs="*", help="File or directory paths to check"
     ).complete = completion.FILE
     parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -1,38 +1,28 @@
 import argparse
-import logging
 import operator
 
-import colorama
-
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 def log_changes(stats):
-    colors = [
-        ("modified", colorama.Fore.YELLOW,),
-        ("added", colorama.Fore.GREEN),
-        ("deleted", colorama.Fore.RED,),
-    ]
+    colors = {
+        "modified": "yellow",
+        "added": "green",
+        "deleted": "red",
+    }
 
-    for state, color in colors:
+    for state, color in colors.items():
         entries = stats.get(state)
 
         if not entries:
             continue
 
         for entry in entries:
-            logger.info(
-                "{color}{state}{nc}{spacing}{entry}".format(
-                    color=color,
-                    state=state[0].upper(),
-                    nc=colorama.Fore.RESET,
-                    spacing="\t",
-                    entry=entry,
-                )
+            ui.write(
+                f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t"
             )
 
 
@@ -58,7 +48,7 @@ class CmdCheckout(CmdBase):
             msg = get_summary(
                 sorted(stats.items(), key=operator.itemgetter(0))
             )
-            logger.info(msg or default_message)
+            ui.write(msg or default_message)
         else:
             log_changes(stats)
 

dvc/command/commit.py

@@ -71,5 +71,5 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="stages or .dvc files to commit. Optional. "
         "(Finds all DVC files in the workspace by default.)",
-    ).complete = completion.DVC_FILE
+    ).complete = completion.DVCFILES_AND_STAGE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -4,6 +4,7 @@ import logging
 import shtab
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 FILE = shtab.FILE
@@ -67,25 +68,16 @@ _dvc_compadd_exps() {
     """,
 }
 
-DVC_FILE = {
-    "bash": "_dvc_compgen_DVCFiles",
-    "zsh": "_dvc_compadd_DVCFiles",
-}
+DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
 
-STAGE = {
-    "bash": "_dvc_compgen_stages",
-    "zsh": "_dvc_compadd_stages",
-}
+STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
 
 DVCFILES_AND_STAGE = {
     "bash": "_dvc_compgen_stages_and_files",
     "zsh": "_dvc_compadd_stages_and_files",
 }
 
-EXPERIMENT = {
-    "bash": "_dvc_compgen_exps",
-    "zsh": "_dvc_compadd_exps",
-}
+EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
 
 
 class CmdCompletion(CmdBaseNoRepo):
@@ -95,7 +87,7 @@ class CmdCompletion(CmdBaseNoRepo):
         parser = get_main_parser()
         shell = self.args.shell
         script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
-        print(script)
+        ui.write(script, force=True)
         return 0
 
 

dvc/command/daemon.py

@@ -67,6 +67,6 @@ def add_parser(subparsers, parent_parser):
         help=DAEMON_ANALYTICS_HELP,
     )
     daemon_analytics_parser.add_argument(
-        "target", help="Analytics file.",
+        "target", help="Analytics file."
     ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -1,9 +1,7 @@
 import argparse
-import logging
 
 from dvc.command.base import CmdBase, append_doc_link
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 def _show_ascii(G):
@@ -101,7 +99,7 @@ class CmdDAG(CmdBase):
         )
 
         if self.args.dot:
-            logger.info(_show_dot(G))
+            ui.write(_show_dot(G))
         else:
             from dvc.utils.pager import pager
 

dvc/command/data_sync.py

@@ -139,7 +139,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
-        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>"
     )
     pull_parser.add_argument(
         "-a",
@@ -156,6 +156,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     pull_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -207,7 +208,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
-        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>"
     )
     push_parser.add_argument(
         "-a",
@@ -224,6 +225,7 @@ def add_parser(subparsers, _parent_parser):
         help="Push cache for all tags.",
     )
     push_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -270,10 +272,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to fetch from",
-        metavar="<name>",
+        "-r", "--remote", help="Remote storage to fetch from", metavar="<name>"
     )
     fetch_parser.add_argument(
         "-a",
@@ -290,6 +289,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     fetch_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -370,6 +370,7 @@ def add_parser(subparsers, _parent_parser):
         "for all tags.",
     )
     status_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/destroy.py

@@ -1,7 +1,7 @@
 import argparse
 import logging
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)

dvc/command/diff.py

@@ -18,9 +18,9 @@ def _digest(checksum):
 
 
 def _show_md(diff, show_hash=False, hide_missing=False):
-    from dvc.utils.diff import table
+    from dvc.ui import ui
 
-    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
+    headers = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
     statuses = ["added", "deleted", "renamed", "modified"]
     if not hide_missing:
@@ -39,7 +39,7 @@ def _show_md(diff, show_hash=False, hide_missing=False):
             else:
                 rows.append([status, path])
 
-    return table(header, rows, True)
+    ui.table(rows, headers=headers, markdown=True)
 
 
 class CmdDiff(CmdBase):

dvc/command/experiments.py

@@ -1,12 +1,13 @@
 import argparse
 import logging
 from collections import Counter, OrderedDict, defaultdict
-from collections.abc import Mapping
 from datetime import date, datetime
-from itertools import groupby
-from typing import Dict, Iterable, Optional
+from fnmatch import fnmatch
+from typing import TYPE_CHECKING, Dict, Iterable, Optional
 
-import dvc.prompt as prompt
+from funcy import lmap
+
+from dvc import prompt
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.command.metrics import DEFAULT_PRECISION
@@ -15,10 +16,17 @@ from dvc.command.repro import add_arguments as add_repro_arguments
 from dvc.exceptions import DvcException, InvalidArgumentError
 from dvc.utils.flatten import flatten
 
+if TYPE_CHECKING:
+    from rich.text import Text
+
+    from dvc.compare import TabularData
+
+
 logger = logging.getLogger(__name__)
 
 
 SHOW_MAX_WIDTH = 1024
+FILL_VALUE = "-"
 
 
 def _filter_name(names, label, filter_strs):
@@ -27,32 +35,23 @@ def _filter_name(names, label, filter_strs):
 
     for filter_s in filter_strs:
         path, _, name = filter_s.rpartition(":")
-        path_filters[path].append(tuple(name.split(".")))
+        path_filters[path].append(name)
 
     for path, filters in path_filters.items():
         if path:
             match_paths = [path]
         else:
             match_paths = names.keys()
-        for length, groups in groupby(filters, len):
-            for group in groups:
-                for match_path in match_paths:
-                    possible_names = [
-                        tuple(name.split(".")) for name in names[match_path]
-                    ]
-                    matches = [
-                        name
-                        for name in possible_names
-                        if name[:length] == group
-                    ]
-                    if not matches:
-                        name = ".".join(group)
-                        raise InvalidArgumentError(
-                            f"'{name}' does not match any known {label}"
-                        )
-                    ret[match_path].update(
-                        {".".join(match): None for match in matches}
+        for match_path in match_paths:
+            for f in filters:
+                matches = [
+                    name for name in names[match_path] if fnmatch(name, f)
+                ]
+                if not matches:
+                    raise InvalidArgumentError(
+                        f"'{f}' does not match any known {label}"
                     )
+                ret[match_path].update({match: None for match in matches})
 
     return ret
 
@@ -103,7 +102,6 @@ def _collect_names(all_experiments, **kwargs):
         for exp in experiments.values():
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
-
     metric_names = _filter_names(
         metric_names,
         "metrics",
@@ -111,7 +109,7 @@ def _collect_names(all_experiments, **kwargs):
         kwargs.get("exclude_metrics"),
     )
     param_names = _filter_names(
-        (param_names),
+        param_names,
         "params",
         kwargs.get("include_params"),
         kwargs.get("exclude_params"),
@@ -120,13 +118,22 @@ def _collect_names(all_experiments, **kwargs):
     return metric_names, param_names
 
 
+experiment_types = {
+    "checkpoint_tip": "│ ╓",
+    "checkpoint_commit": "│ ╟",
+    "checkpoint_base": "├─╨",
+    "branch_commit": "├──",
+    "branch_base": "└──",
+    "baseline": "",
+}
+
+
 def _collect_rows(
     base_rev,
     experiments,
     metric_names,
     param_names,
     precision=DEFAULT_PRECISION,
-    no_timestamp=False,
     sort_by=None,
     sort_order=None,
 ):
@@ -143,56 +150,56 @@ def _collect_rows(
 
     new_checkpoint = True
     for i, (rev, exp) in enumerate(experiments.items()):
-        row = []
-        style = None
-        queued = "*" if exp.get("queued", False) else ""
+        queued = str(exp.get("queued") or "")
+        is_baseline = rev == "baseline"
 
-        tip = exp.get("checkpoint_tip")
-        parent = ""
-        if rev == "baseline":
-            if Git.is_sha(base_rev):
-                name_rev = base_rev[:7]
-            else:
-                name_rev = base_rev
-            name = exp.get("name", name_rev)
-            row.append(f"{name}")
-            style = "bold"
+        if is_baseline:
+            name_rev = base_rev[:7] if Git.is_sha(base_rev) else base_rev
         else:
-            if tip:
-                parent_rev = exp.get("checkpoint_parent", "")
-                parent_exp = experiments.get(parent_rev, {})
-                parent_tip = parent_exp.get("checkpoint_tip")
-                if tip == parent_tip:
-                    if new_checkpoint:
-                        tree = "│ ╓"
-                    else:
-                        tree = "│ ╟"
-                    new_checkpoint = False
-                else:
-                    if parent_rev == base_rev:
-                        tree = "├─╨"
-                    else:
-                        tree = "│ ╟"
-                        parent = f" ({parent_rev[:7]})"
-                    new_checkpoint = True
+            name_rev = rev[:7]
+
+        exp_name = exp.get("name", "")
+        tip = exp.get("checkpoint_tip")
+
+        parent_rev = exp.get("checkpoint_parent", "")
+        parent_exp = experiments.get(parent_rev, {})
+        parent_tip = parent_exp.get("checkpoint_tip")
+
+        parent = ""
+        if is_baseline:
+            typ = "baseline"
+        elif tip:
+            if tip == parent_tip:
+                typ = (
+                    "checkpoint_tip" if new_checkpoint else "checkpoint_commit"
+                )
+            elif parent_rev == base_rev:
+                typ = "checkpoint_base"
             else:
-                if i < len(experiments) - 1:
-                    tree = "├──"
-                else:
-                    tree = "└──"
-                new_checkpoint = True
-            name = exp.get("name", rev[:7])
-            row.append(f"{tree} {queued}{name}{parent}")
+                typ = "checkpoint_commit"
+                parent = parent_rev[:7]
+        elif i < len(experiments) - 1:
+            typ = "branch_commit"
+        else:
+            typ = "branch_base"
 
-        if not no_timestamp:
-            row.append(_format_time(exp.get("timestamp")))
+        if not is_baseline:
+            new_checkpoint = not (tip and tip == parent_tip)
 
+        row = [
+            exp_name,
+            name_rev,
+            queued,
+            typ,
+            _format_time(exp.get("timestamp")),
+            parent,
+        ]
         _extend_row(
             row, metric_names, exp.get("metrics", {}).items(), precision
         )
         _extend_row(row, param_names, exp.get("params", {}).items(), precision)
 
-        yield row, style
+        yield row
 
 
 def _sort_column(sort_by, metric_names, param_names):
@@ -233,7 +240,7 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
             return _sort((tip, experiments[tip]))
         data = exp.get(typ, {}).get(sort_path, {})
         val = flatten(data).get(sort_name)
-        return (val is None, val)
+        return val is None, val
 
     ret = OrderedDict()
     if "baseline" in experiments:
@@ -245,7 +252,7 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
 
 def _format_time(timestamp):
     if timestamp is None:
-        return "-"
+        return FILL_VALUE
     if timestamp.date() == date.today():
         fmt = "%I:%M %p"
     else:
@@ -253,42 +260,23 @@ def _format_time(timestamp):
     return timestamp.strftime(fmt)
 
 
-def _format_field(val, precision=DEFAULT_PRECISION):
-    if isinstance(val, float):
-        fmt = f"{{:.{precision}g}}"
-        return fmt.format(val)
-    if isinstance(val, Mapping):
-        return {k: _format_field(v) for k, v in val.items()}
-    if isinstance(val, list):
-        return [_format_field(x) for x in val]
-    return str(val)
-
-
 def _extend_row(row, names, items, precision):
     from rich.text import Text
 
+    from dvc.compare import _format_field, with_value
+
     if not items:
-        row.extend(["-"] * len(names))
+        row.extend(FILL_VALUE for keys in names.values() for _ in keys)
         return
 
     for fname, item in items:
-        if isinstance(item, dict):
-            item = flatten(item)
-        else:
-            item = {fname: item}
+        item = flatten(item) if isinstance(item, dict) else {fname: item}
         for name in names[fname]:
-            if name in item:
-                value = item[name]
-                if value is None:
-                    text = "-"
-                else:
-                    # wrap field data in rich.Text, otherwise rich may
-                    # interpret unescaped braces from list/dict types as rich
-                    # markup tags
-                    text = Text(str(_format_field(value, precision)))
-                row.append(text)
-            else:
-                row.append("-")
+            value = with_value(item.get(name), FILL_VALUE)
+            # wrap field data in rich.Text, otherwise rich may
+            # interpret unescaped braces from list/dict types as rich
+            # markup tags
+            row.append(Text(str(_format_field(value, precision))))
 
 
 def _parse_filter_list(param_list):
@@ -302,9 +290,68 @@ def _parse_filter_list(param_list):
     return ret
 
 
-def _experiments_table(all_experiments, **kwargs):
-    from dvc.utils.table import Table
+def experiments_table(
+    all_experiments,
+    metric_headers,
+    metric_names,
+    param_headers,
+    param_names,
+    sort_by=None,
+    sort_order=None,
+    precision=DEFAULT_PRECISION,
+) -> "TabularData":
+    from funcy import lconcat
 
+    from dvc.compare import TabularData
+
+    headers = ["Experiment", "rev", "queued", "typ", "Created", "parent"]
+    td = TabularData(
+        lconcat(headers, metric_headers, param_headers), fill_value=FILL_VALUE
+    )
+    for base_rev, experiments in all_experiments.items():
+        rows = _collect_rows(
+            base_rev,
+            experiments,
+            metric_names,
+            param_names,
+            sort_by=sort_by,
+            sort_order=sort_order,
+            precision=precision,
+        )
+        td.extend(rows)
+
+    return td
+
+
+def prepare_exp_id(kwargs) -> "Text":
+    from rich.text import Text
+
+    exp_name = kwargs["Experiment"]
+    rev = kwargs["rev"]
+    typ = kwargs.get("typ", "baseline")
+
+    if typ == "baseline" or not exp_name:
+        text = Text(exp_name or rev)
+    else:
+        text = Text.assemble(rev, " [", (exp_name, "bold"), "]")
+
+    parent = kwargs.get("parent")
+    suff = f" ({parent})" if parent else ""
+    text.append(suff)
+
+    tree = experiment_types[typ]
+    queued = "*" if kwargs.get("queued") else ""
+    pref = (f"{tree} " if tree else "") + queued
+    return Text(pref) + text
+
+
+def baseline_styler(typ):
+    return {"style": "bold"} if typ == "baseline" else {}
+
+
+def show_experiments(
+    all_experiments, pager=True, no_timestamp=False, **kwargs
+):
     include_metrics = _parse_filter_list(kwargs.pop("include_metrics", []))
     exclude_metrics = _parse_filter_list(kwargs.pop("exclude_metrics", []))
     include_params = _parse_filter_list(kwargs.pop("include_params", []))
@@ -317,44 +364,66 @@ def _experiments_table(all_experiments, **kwargs):
         include_params=include_params,
         exclude_params=exclude_params,
     )
+    metric_headers = _normalize_headers(metric_names)
+    param_headers = _normalize_headers(param_names)
 
-    table = Table()
-    table.add_column(
-        "Experiment", no_wrap=True, header_style="black on grey93"
-    )
-    if not kwargs.get("no_timestamp", False):
-        table.add_column("Created", header_style="black on grey93")
-    _add_data_columns(
-        table,
+    td = experiments_table(
+        all_experiments,
+        metric_headers,
         metric_names,
-        justify="right",
-        no_wrap=True,
-        header_style="black on cornsilk1",
-    )
-    _add_data_columns(
-        table, param_names, justify="left", header_style="black on light_cyan1"
+        param_headers,
+        param_names,
+        kwargs.get("sort_by"),
+        kwargs.get("sort_order"),
+        kwargs.get("precision"),
     )
 
-    for base_rev, experiments in all_experiments.items():
-        for row, _, in _collect_rows(
-            base_rev, experiments, metric_names, param_names, **kwargs,
-        ):
-            table.add_row(*row)
+    if no_timestamp:
+        td.drop("Created")
 
-    return table
+    row_styles = lmap(baseline_styler, td.column("typ"))
+
+    merge_headers = ["Experiment", "rev", "queued", "typ", "parent"]
+    td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers))
+    td.drop(*merge_headers[1:])
+
+    headers = {"metrics": metric_headers, "params": param_headers}
+    styles = {
+        "Experiment": {"no_wrap": True, "header_style": "black on grey93"},
+        "Created": {"header_style": "black on grey93"},
+    }
+    header_bg_colors = {"metrics": "cornsilk1", "params": "light_cyan1"}
+    styles.update(
+        {
+            header: {
+                "justify": "left" if typ == "metrics" else "params",
+                "header_style": f"black on {header_bg_colors[typ]}",
+                "collapse": idx != 0,
+                "no_wrap": typ == "metrics",
+            }
+            for typ, hs in headers.items()
+            for idx, header in enumerate(hs)
+        }
+    )
+
+    td.render(
+        pager=pager,
+        borders=True,
+        rich_table=True,
+        header_styles=styles,
+        row_styles=row_styles,
+    )
 
 
-def _add_data_columns(table, names, **kwargs):
+def _normalize_headers(names):
     count = Counter(
         name for path in names for name in names[path] for path in names
     )
-    first = True
-    for path in names:
-        for name in names[path]:
-            col_name = name if count[name] == 1 else f"{path}:{name}"
-            kwargs["collapse"] = False if first else True
-            table.add_column(col_name, **kwargs)
-            first = False
+    return [
+        name if count[name] == 1 else f"{path}:{name}"
+        for path in names
+        for name in names[path]
+    ]
 
 
 def _format_json(item):
@@ -365,8 +434,6 @@ def _format_json(item):
 
 class CmdExperimentsShow(CmdBase):
     def run(self):
-        from rich.console import Console
-
         try:
             all_experiments = self.repo.experiments.show(
                 all_branches=self.args.all_branches,
@@ -374,20 +441,18 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 sha_only=self.args.sha,
                 num=self.args.num,
+                param_deps=self.args.param_deps,
             )
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
 
-            if self.args.show_json:
-                import json
+        if self.args.show_json:
+            import json
 
-                logger.info(json.dumps(all_experiments, default=_format_json))
-                return 0
-
-            if self.args.precision is None:
-                precision = DEFAULT_PRECISION
-            else:
-                precision = self.args.precision
-
-            table = _experiments_table(
+            logger.info(json.dumps(all_experiments, default=_format_json))
+        else:
+            show_experiments(
                 all_experiments,
                 include_metrics=self.args.include_metrics,
                 exclude_metrics=self.args.exclude_metrics,
@@ -396,29 +461,9 @@ class CmdExperimentsShow(CmdBase):
                 no_timestamp=self.args.no_timestamp,
                 sort_by=self.args.sort_by,
                 sort_order=self.args.sort_order,
-                precision=precision,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                pager=not self.args.no_pager,
             )
-
-            console = Console()
-            if self.args.no_pager:
-                console.print(table)
-            else:
-                from dvc.utils.pager import DvcPager
-
-                # NOTE: rich does not have native support for unlimited width
-                # via pager. we override rich table compression by setting
-                # console width to the full width of the table
-                measurement = table.__rich_measure__(console, SHOW_MAX_WIDTH)
-                console._width = (  # pylint: disable=protected-access
-                    measurement.maximum
-                )
-                with console.pager(pager=DvcPager(), styles=True):
-                    console.print(table)
-
-        except DvcException:
-            logger.exception("failed to show experiments")
-            return 1
-
         return 0
 
 
@@ -432,43 +477,6 @@ class CmdExperimentsApply(CmdBase):
         return 0
 
 
-def _show_diff(
-    diff,
-    title="",
-    markdown=False,
-    no_path=False,
-    old=False,
-    precision=DEFAULT_PRECISION,
-):
-    from dvc.utils.diff import table
-
-    rows = []
-    for fname, diff_ in diff.items():
-        sorted_diff = OrderedDict(sorted(diff_.items()))
-        for item, change in sorted_diff.items():
-            row = [] if no_path else [fname]
-            row.append(item)
-            if old:
-                row.append(_format_field(change.get("old"), precision))
-            row.append(_format_field(change["new"], precision))
-            row.append(
-                _format_field(
-                    change.get("diff", "diff not supported"), precision
-                )
-            )
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append(title)
-    if old:
-        header.extend(["Old", "New"])
-    else:
-        header.append("Value")
-    header.append("Change")
-
-    return table(header, rows, markdown)
-
-
 class CmdExperimentsDiff(CmdBase):
     def run(self):
 
@@ -477,47 +485,56 @@ class CmdExperimentsDiff(CmdBase):
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
                 all=self.args.all,
+                param_deps=self.args.param_deps,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                if self.args.precision is None:
-                    precision = DEFAULT_PRECISION
-                else:
-                    precision = self.args.precision
-
-                diffs = [("metrics", "Metric"), ("params", "Param")]
-                for key, title in diffs:
-                    table = _show_diff(
-                        diff[key],
-                        title=title,
-                        markdown=self.args.show_md,
-                        no_path=self.args.no_path,
-                        old=self.args.old,
-                        precision=precision,
-                    )
-                    if table:
-                        logger.info(table)
-                        logger.info("")
-
         except DvcException:
             logger.exception("failed to show experiments diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            precision = self.args.precision or DEFAULT_PRECISION
+            diffs = [("metrics", "Metric"), ("params", "Param")]
+            for idx, (key, title) in enumerate(diffs):
+                if idx:
+                    from dvc.ui import ui
+
+                    # we are printing tables even in `--quiet` mode
+                    # so we should also be printing the "table" separator
+                    ui.write(force=True)
+
+                show_diff(
+                    diff[key],
+                    title=title,
+                    markdown=self.args.show_md,
+                    no_path=self.args.no_path,
+                    old=self.args.old,
+                    on_empty_diff="diff not supported",
+                    precision=precision if key == "metrics" else None,
+                )
+
         return 0
 
 
 class CmdExperimentsRun(CmdRepro):
     def run(self):
-        from dvc.command.metrics import _show_metrics
+        from dvc.compare import show_metrics
 
-        if self.args.reset and self.args.checkpoint_resume:
-            raise InvalidArgumentError(
-                "--reset and --rev are mutually exclusive."
-            )
+        if self.args.checkpoint_resume:
+            if self.args.reset:
+                raise InvalidArgumentError(
+                    "--reset and --rev are mutually exclusive."
+                )
+            if not (self.args.queue or self.args.tmp_dir):
+                raise InvalidArgumentError(
+                    "--rev can only be used in conjunction with "
+                    "--queue or --temp."
+                )
 
         if self.args.reset:
             logger.info("Any existing checkpoints will be reset and re-run.")
@@ -537,7 +554,7 @@ class CmdExperimentsRun(CmdRepro):
         if self.args.metrics and results:
             metrics = self.repo.metrics.show(revs=list(results))
             metrics.pop("workspace", None)
-            logger.info(_show_metrics(metrics))
+            logger.info(show_metrics(metrics))
 
         return 0
 
@@ -690,7 +707,7 @@ class CmdExperimentsRemove(CmdBase):
     def run(self):
 
         self.repo.experiments.remove(
-            exp_names=self.args.experiment, queue=self.args.queue,
+            exp_names=self.args.experiment, queue=self.args.queue
         )
 
         return 0
@@ -788,6 +805,12 @@ def add_parser(subparsers, parent_parser):
         help="Exclude the specified params from output table.",
         metavar="<params_list>",
     )
+    experiments_show_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_show_parser.add_argument(
         "--sort-by",
         help="Sort related experiments by the specified metric or param.",
@@ -845,7 +868,7 @@ def add_parser(subparsers, parent_parser):
         help="Fail if this command would overwrite conflicting changes.",
     )
     experiments_apply_parser.add_argument(
-        "experiment", help="Experiment to be applied.",
+        "experiment", help="Experiment to be applied."
     ).complete = completion.EXPERIMENT
     experiments_apply_parser.set_defaults(func=CmdExperimentsApply)
 
@@ -873,6 +896,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged metrics/params as well.",
     )
+    experiments_diff_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_diff_parser.add_argument(
         "--show-json",
         action="store_true",
@@ -925,9 +954,8 @@ def add_parser(subparsers, parent_parser):
         type=str,
         dest="checkpoint_resume",
         help=(
-            "Continue the specified checkpoint experiment. "
-            "(Only required for explicitly resuming checkpoints in queued "
-            "or temp dir runs.)"
+            "Continue the specified checkpoint experiment. Can only be used "
+            "in conjunction with --queue or --temp."
         ),
         metavar="<experiment_rev>",
     ).complete = completion.EXPERIMENT
@@ -972,6 +1000,7 @@ def add_parser(subparsers, parent_parser):
         help="Keep experiments derived from all Git tags.",
     )
     experiments_gc_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -1004,10 +1033,10 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_branch_parser.add_argument(
-        "experiment", help="Experiment to be promoted.",
+        "experiment", help="Experiment to be promoted."
     )
     experiments_branch_parser.add_argument(
-        "branch", help="Git branch name to use.",
+        "branch", help="Git branch name to use."
     )
     experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)
 
@@ -1030,7 +1059,7 @@ def add_parser(subparsers, parent_parser):
         metavar="<rev>",
     )
     experiments_list_parser.add_argument(
-        "--all", action="store_true", help="List all experiments.",
+        "--all", action="store_true", help="List all experiments."
     )
     experiments_list_parser.add_argument(
         "--names-only",
@@ -1102,7 +1131,7 @@ def add_parser(subparsers, parent_parser):
         metavar="<git_remote>",
     )
     experiments_push_parser.add_argument(
-        "experiment", help="Experiment to push.", metavar="<experiment>",
+        "experiment", help="Experiment to push.", metavar="<experiment>"
     ).complete = completion.EXPERIMENT
     experiments_push_parser.set_defaults(func=CmdExperimentsPush)
 
@@ -1158,7 +1187,7 @@ def add_parser(subparsers, parent_parser):
         metavar="<git_remote>",
     )
     experiments_pull_parser.add_argument(
-        "experiment", help="Experiment to pull.", metavar="<experiment>",
+        "experiment", help="Experiment to pull.", metavar="<experiment>"
     )
     experiments_pull_parser.set_defaults(func=CmdExperimentsPull)
 
@@ -1171,7 +1200,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_remove_parser.add_argument(
-        "--queue", action="store_true", help="Remove all queued experiments.",
+        "--queue", action="store_true", help="Remove all queued experiments."
     )
     experiments_remove_parser.add_argument(
         "experiment",
@@ -1220,6 +1249,7 @@ def _add_run_common(parser):
         "-j",
         "--jobs",
         type=int,
+        default=1,
         help="Run the specified number of experiments at a time in parallel.",
         metavar="<number>",
     )

dvc/command/freeze.py

@@ -40,7 +40,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to freeze",
+        "targets", nargs="+", help="Stages or .dvc files to freeze"
     ).complete = completion.DVC_FILE
     freeze_parser.set_defaults(func=CmdFreeze)
 
@@ -53,6 +53,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze"
     ).complete = completion.DVC_FILE
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 import os
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)
@@ -97,6 +97,7 @@ def add_parser(subparsers, parent_parser):
         help="Keep data files for all Git tags.",
     )
     gc_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/get.py

@@ -64,7 +64,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
+        "path", help="Path to a file or directory within the repository"
     ).complete = completion.FILE
     get_parser.add_argument(
         "-o",

dvc/command/get_url.py

@@ -34,7 +34,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to.",
+        "out", nargs="?", help="Destination path to put data to."
     ).complete = completion.DIR
     get_parser.add_argument(
         "-j",

dvc/command/git_hook.py

@@ -105,7 +105,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_COMMIT_HELP,
     )
     pre_commit_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     pre_commit_parser.set_defaults(func=CmdPreCommit)
 
@@ -117,7 +117,7 @@ def add_parser(subparsers, parent_parser):
         help=POST_CHECKOUT_HELP,
     )
     post_checkout_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     post_checkout_parser.set_defaults(func=CmdPostCheckout)
 
@@ -129,7 +129,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_PUSH_HELP,
     )
     pre_push_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
 
@@ -146,9 +146,7 @@ def add_parser(subparsers, parent_parser):
         help="Ancestor's version of the conflicting file.",
     )
     merge_driver_parser.add_argument(
-        "--our",
-        required=True,
-        help="Current version of the conflicting file.",
+        "--our", required=True, help="Current version of the conflicting file."
     )
     merge_driver_parser.add_argument(
         "--their",

dvc/command/imp.py

@@ -48,7 +48,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
+        "path", help="Path to a file or directory within the repository"
     ).complete = completion.FILE
     import_parser.add_argument(
         "-o",

dvc/command/imp_url.py

@@ -58,7 +58,7 @@ def add_parser(subparsers, parent_parser):
         "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to.",
+        "out", nargs="?", help="Destination path to put files to."
     ).complete = completion.DIR
     import_parser.add_argument(
         "--file",

dvc/command/init.py

@@ -39,14 +39,14 @@ class CmdInit(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            self.repo = Repo.init(
+            with Repo.init(
                 ".",
                 no_scm=self.args.no_scm,
                 force=self.args.force,
                 subdir=self.args.subdir,
-            )
-            self.config = self.repo.config
-            _welcome_message()
+            ) as repo:
+                self.config = repo.config
+                _welcome_message()
         except InitError:
             logger.exception("failed to initiate DVC")
             return 1

dvc/command/live.py

@@ -4,7 +4,6 @@ import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, fix_subparsers
-from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +15,7 @@ class CmdLive(CmdBase):
         metrics, plots = self.repo.live.show(target=target, revs=revs)
 
         html_path = self.args.target + ".html"
-        write(html_path, plots, metrics)
+        self.repo.plots.write_html(html_path, plots, metrics)
 
         logger.info(f"\nfile://{os.path.abspath(html_path)}")
 
@@ -36,7 +35,7 @@ class CmdLiveDiff(CmdLive):
 def shared_parent_parser():
     parent_parser = argparse.ArgumentParser(add_help=False)
     parent_parser.add_argument(
-        "target", help="Logs dir to produce summary from",
+        "target", help="Logs dir to produce summary from"
     ).complete = completion.DIR
     parent_parser.add_argument(
         "-o",

dvc/command/ls/__init__.py

@@ -1,11 +1,11 @@
 import argparse
 import logging
-import sys
 
 from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -34,9 +34,13 @@ class CmdList(CmdBaseNoRepo):
                 recursive=self.args.recursive,
                 dvc_only=self.args.dvc_only,
             )
-            if entries:
-                entries = _prettify(entries, sys.stdout.isatty())
-                logger.info("\n".join(entries))
+            if self.args.show_json:
+                import json
+
+                ui.write(json.dumps(entries))
+            elif entries:
+                entries = _prettify(entries, with_color=True)
+                ui.write("\n".join(entries))
             return 0
         except DvcException:
             logger.exception(f"failed to list '{self.args.url}'")
@@ -65,6 +69,9 @@ def add_parser(subparsers, parent_parser):
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
+    list_parser.add_argument(
+        "--show-json", action="store_true", help="Show output in JSON format."
+    )
     list_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/metrics.py

@@ -11,64 +11,6 @@ logger = logging.getLogger(__name__)
 DEFAULT_PRECISION = 5
 
 
-def _show_metrics(
-    metrics,
-    markdown=False,
-    all_branches=False,
-    all_tags=False,
-    all_commits=False,
-    precision=None,
-):
-    from dvc.utils.diff import format_dict, table
-    from dvc.utils.flatten import flatten
-
-    # When `metrics` contains a `None` key, it means that some files
-    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
-    missing = metrics.pop(None, None)
-    with_rev = any([all_branches, all_tags, all_commits])
-    header_set = set()
-    rows = []
-
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-        return val
-
-    for _branch, val in metrics.items():
-        for _fname, metric in val.items():
-            if not isinstance(metric, dict):
-                header_set.add("")
-                continue
-            for key, _val in flatten(format_dict(metric)).items():
-                header_set.add(key)
-    header = sorted(header_set)
-    for branch, val in metrics.items():
-        for fname, metric in val.items():
-            row = []
-            if with_rev:
-                row.append(branch)
-            row.append(fname)
-            if not isinstance(metric, dict):
-                row.append(str(metric))
-                rows.append(row)
-                continue
-            flattened_val = flatten(format_dict(metric))
-
-            for i in header:
-                row.append(_round(flattened_val.get(i)))
-            rows.append(row)
-    header.insert(0, "Path")
-    if with_rev:
-        header.insert(0, "Revision")
-
-    if missing:
-        raise BadMetricError(missing)
-    return table(header, rows, markdown)
-
-
 class CmdMetricsBase(CmdBase):
     UNINITIALIZED = True
 
@@ -83,61 +25,37 @@ class CmdMetricsShow(CmdMetricsBase):
                 all_commits=self.args.all_commits,
                 recursive=self.args.recursive,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(metrics))
-            else:
-                table = _show_metrics(
-                    metrics,
-                    self.args.show_md,
-                    self.args.all_branches,
-                    self.args.all_tags,
-                    self.args.all_commits,
-                )
-                if table:
-                    logger.info(table)
         except DvcException:
             logger.exception("")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(metrics))
+        else:
+            from dvc.compare import show_metrics
+
+            # When `metrics` contains a `None` key, it means that some files
+            # specified as `targets` in `repo.metrics.show` didn't contain any
+            # metrics.
+            missing = metrics.pop(None, None)
+            if missing:
+                raise BadMetricError(missing)
+
+            show_metrics(
+                metrics,
+                markdown=self.args.show_md,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
-def _show_diff(diff, markdown=False, no_path=False, precision=None):
-    from collections import OrderedDict
-
-    from dvc.utils.diff import table
-
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-
-        return val
-
-    rows = []
-    for fname, mdiff in diff.items():
-        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
-        for metric, change in sorted_mdiff.items():
-            row = [] if no_path else [fname]
-            row.append(metric)
-            row.append(_round(change.get("old")))
-            row.append(_round(change["new"]))
-            row.append(_round(change.get("diff")))
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Metric")
-    header.extend(["Old", "New"])
-    header.append("Change")
-
-    return table(header, rows, markdown)
-
-
 class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
         try:
@@ -148,25 +66,26 @@ class CmdMetricsDiff(CmdMetricsBase):
                 recursive=self.args.recursive,
                 all=self.args.all,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(
-                    diff,
-                    self.args.show_md,
-                    self.args.no_path,
-                    precision=self.args.precision,
-                )
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show metrics diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Metric",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
@@ -219,6 +138,7 @@ def add_parser(subparsers, parent_parser):
         help="Show metrics for all tags.",
     )
     metrics_show_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/move.py

@@ -37,9 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory.",
+        "src", help="Source path to a data file or directory."
     ).complete = completion.FILE
     move_parser.add_argument(
-        "dst", help="Destination path.",
+        "dst", help="Destination path."
     ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/params.py

@@ -1,6 +1,5 @@
 import argparse
 import logging
-from collections import OrderedDict
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
@@ -9,27 +8,6 @@ from dvc.exceptions import DvcException
 logger = logging.getLogger(__name__)
 
 
-def _show_diff(diff, markdown=False, no_path=False):
-    from dvc.utils.diff import table
-
-    rows = []
-    for fname, pdiff in diff.items():
-        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
-        for param, change in sorted_pdiff.items():
-            row = [] if no_path else [fname]
-            row.append(param)
-            row.append(change["old"])
-            row.append(change["new"])
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Param")
-    header.append("Old")
-    header.append("New")
-
-    return table(header, rows, markdown)
-
-
 class CmdParamsDiff(CmdBase):
     UNINITIALIZED = True
 
@@ -40,21 +18,27 @@ class CmdParamsDiff(CmdBase):
                 b_rev=self.args.b_rev,
                 targets=self.args.targets,
                 all=self.args.all,
+                deps=self.args.deps,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(diff, self.args.show_md, self.args.no_path)
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show params diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Param",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                show_changes=False,
+            )
+
         return 0
 
 
@@ -111,6 +95,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged params as well.",
     )
+    params_diff_parser.add_argument(
+        "--deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     params_diff_parser.add_argument(
         "--show-json",
         action="store_true",

dvc/command/plots.py

@@ -5,7 +5,6 @@ from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 from dvc.utils import format_link
-from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +43,10 @@ class CmdPlots(CmdBase):
 
             rel: str = self.args.out or "plots.html"
             path = (Path.cwd() / rel).resolve()
-            write(path, plots)
+            self.repo.plots.write_html(
+                path, plots=plots, html_template_path=self.args.html_template
+            )
+
         except DvcException:
             logger.exception("")
             return 1
@@ -85,7 +87,7 @@ class CmdPlotsDiff(CmdPlots):
 class CmdPlotsModify(CmdPlots):
     def run(self):
         self.repo.plots.modify(
-            self.args.target, props=self._props(), unset=self.args.unset,
+            self.args.target, props=self._props(), unset=self.args.unset
         )
         return 0
 
@@ -93,7 +95,7 @@ class CmdPlotsModify(CmdPlots):
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Commands to visualize and compare plot metrics in structured files "
-        "(JSON, YAML, CSV, TSV)"
+        "(JSON, YAML, CSV, TSV)."
     )
 
     plots_parser = subparsers.add_parser(
@@ -158,7 +160,7 @@ def add_parser(subparsers, parent_parser):
         help=argparse.SUPPRESS,
     )
     plots_diff_parser.add_argument(
-        "revisions", nargs="*", default=None, help="Git commits to plot from",
+        "revisions", nargs="*", default=None, help="Git commits to plot from"
     )
     _add_props_arguments(plots_diff_parser)
     _add_output_arguments(plots_diff_parser)
@@ -173,7 +175,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
-        "target", help="Metric file to set properties to",
+        "target", help="Metric file to set properties to"
     ).complete = completion.FILE
     _add_props_arguments(plots_modify_parser)
     plots_modify_parser.add_argument(
@@ -243,3 +245,9 @@ def _add_output_arguments(parser):
         default=False,
         help="Open plot file directly in the browser.",
     )
+    parser.add_argument(
+        "--html-template",
+        default=None,
+        help="Custom HTML template for VEGA visualization.",
+        metavar="<path>",
+    )

dvc/command/repro.py

@@ -3,7 +3,6 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import _show_metrics
 from dvc.command.status import CmdDataStatus
 
 logger = logging.getLogger(__name__)
@@ -20,8 +19,10 @@ class CmdRepro(CmdBase):
             )
 
         if self.args.metrics:
+            from dvc.compare import show_metrics
+
             metrics = self.repo.metrics.show()
-            logger.info(_show_metrics(metrics))
+            show_metrics(metrics)
 
         return 0
 

dvc/command/run.py

@@ -33,36 +33,17 @@ class CmdRun(CmdBase):
             )
             return 1
 
+        kwargs = vars(self.args)
+        kwargs.update(
+            {
+                "cmd": parse_cmd(self.args.cmd),
+                "fname": kwargs.pop("file"),
+                "no_exec": (self.args.no_exec or bool(self.args.checkpoints)),
+                "run_cache": not kwargs.pop("no_run_cache"),
+            }
+        )
         try:
-            self.repo.run(
-                cmd=parse_cmd(self.args.cmd),
-                outs=self.args.outs,
-                outs_no_cache=self.args.outs_no_cache,
-                metrics=self.args.metrics,
-                metrics_no_cache=self.args.metrics_no_cache,
-                plots=self.args.plots,
-                plots_no_cache=self.args.plots_no_cache,
-                live=self.args.live,
-                live_no_cache=self.args.live_no_cache,
-                live_no_summary=self.args.live_no_summary,
-                live_no_html=self.args.live_no_html,
-                deps=self.args.deps,
-                params=self.args.params,
-                fname=self.args.file,
-                wdir=self.args.wdir,
-                no_exec=(self.args.no_exec or bool(self.args.checkpoints)),
-                force=self.args.force,
-                run_cache=not self.args.no_run_cache,
-                no_commit=self.args.no_commit,
-                outs_persist=self.args.outs_persist,
-                outs_persist_no_cache=self.args.outs_persist_no_cache,
-                checkpoints=self.args.checkpoints,
-                always_changed=self.args.always_changed,
-                name=self.args.name,
-                single_stage=self.args.single_stage,
-                external=self.args.external,
-                desc=self.args.desc,
-            )
+            self.repo.run(**kwargs)
         except DvcException:
             logger.exception("")
             return 1
@@ -83,11 +64,9 @@ def add_parser(subparsers, parent_parser):
         help=RUN_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    run_parser.add_argument("-n", "--name", help="Stage name.")
     run_parser.add_argument(
-        "-n", "--name", help="Stage name.",
-    )
-    run_parser.add_argument(
-        "--file", metavar="<filename>", help=argparse.SUPPRESS,
+        "--file", metavar="<filename>", help=argparse.SUPPRESS
     )
     run_parser.add_argument(
         "--no-exec",

dvc/command/stage.py

@@ -9,7 +9,7 @@ from dvc.utils.cli_parse import parse_params
 from dvc.utils.humanize import truncate_text
 
 if TYPE_CHECKING:
-    from dvc.output.base import BaseOutput
+    from dvc.output import Output
     from dvc.stage import Stage
 
 logger = logging.getLogger(__name__)
@@ -18,7 +18,7 @@ MAX_TEXT_LENGTH = 80
 
 
 def generate_description(stage: "Stage") -> str:
-    def part_desc(outs: Iterable["BaseOutput"]) -> str:
+    def part_desc(outs: Iterable["Output"]) -> str:
         return ", ".join(out.def_path for out in outs)
 
     if not stage.deps and not stage.outs:
@@ -27,7 +27,7 @@ def generate_description(stage: "Stage") -> str:
     if not stage.outs and stage.deps:
         return "Depends on " + part_desc(stage.deps)
 
-    def is_plot_or_metric(out: "BaseOutput"):
+    def is_plot_or_metric(out: "Output"):
         return bool(out.plot) or bool(out.metric)
 
     desc: List[str] = []
@@ -75,16 +75,14 @@ class CmdStageList(CmdBase):
         # removing duplicates while maintaining order
         collected = chain.from_iterable(
             self.repo.stage.collect(
-                target=target,
-                recursive=self.args.recursive,
-                accept_group=True,
+                target=target, recursive=self.args.recursive, accept_group=True
             )
             for target in self.args.targets
         )
         return dict.fromkeys(collected).keys()
 
     def run(self):
-        from dvc.utils.diff import table
+        from dvc.ui import ui
 
         def log_error(relpath: str, exc: Exception):
             if self.args.fail:
@@ -98,8 +96,7 @@ class CmdStageList(CmdBase):
         names_only = self.args.names_only
 
         data = prepare_stages_data(stages, description=not names_only)
-        if data:
-            print(table(header=(), rows=data.items()))
+        ui.table(data.items())
 
         return 0
 
@@ -116,8 +113,11 @@ def parse_cmd(commands: List[str]) -> str:
     """
 
     def quote_argument(arg: str):
-        should_quote = " " in arg and '"' not in arg
-        return f'"{arg}"' if should_quote else arg
+        if not arg:
+            return '""'
+        if " " in arg and '"' not in arg:
+            return f'"{arg}"'
+        return arg
 
     if len(commands) < 2:
         return " ".join(commands)
@@ -202,7 +202,7 @@ def _add_common_args(parser):
         metavar="<path>",
     )
     parser.add_argument(
-        "--live", help="Declare output as dvclive.", metavar="<path>",
+        "--live", help="Declare output as dvclive.", metavar="<path>"
     )
     parser.add_argument(
         "--live-no-cache",

dvc/command/status.py

@@ -44,21 +44,27 @@ class CmdDataStatus(CmdDataBase):
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.repo import lock_repo
         from dvc.ui import ui
 
         indent = 1 if self.args.cloud else 0
-        try:
-            st = self.repo.status(
-                targets=self.args.targets,
-                jobs=self.args.jobs,
-                cloud=self.args.cloud,
-                remote=self.args.remote,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                with_deps=self.args.with_deps,
-                recursive=self.args.recursive,
-            )
+
+        with lock_repo(self.repo):
+            try:
+                st = self.repo.status(
+                    targets=self.args.targets,
+                    jobs=self.args.jobs,
+                    cloud=self.args.cloud,
+                    remote=self.args.remote,
+                    all_branches=self.args.all_branches,
+                    all_tags=self.args.all_tags,
+                    all_commits=self.args.all_commits,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("")
+                return 1
 
             if self.args.quiet:
                 return bool(st)
@@ -67,9 +73,14 @@ class CmdDataStatus(CmdDataBase):
                 import json
 
                 logger.info(json.dumps(st))
-            elif st:
+                return 0
+
+            if st:
                 self._show(st, indent)
-            elif not self.repo.stages:
+                return 0
+
+            # additional hints for the user
+            if not self.repo.stages:
                 ui.write(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
@@ -79,7 +90,4 @@ class CmdDataStatus(CmdDataBase):
             else:
                 ui.write(self.UP_TO_DATE_MSG)
 
-        except DvcException:
-            logger.exception("")
-            return 1
         return 0

dvc/command/unprotect.py

@@ -23,7 +23,7 @@ class CmdUnprotect(CmdBase):
 def add_parser(subparsers, parent_parser):
     UNPROTECT_HELP = (
         "Unprotect tracked files or directories (when hardlinks or symlinks "
-        "have been enabled with `dvc config cache.type`)"
+        "have been enabled with `dvc config cache.type`)."
     )
     unprotect_parser = subparsers.add_parser(
         "unprotect",
@@ -33,6 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect.",
+        "targets", nargs="+", help="Data files/directories to unprotect."
     ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -36,7 +36,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help=".dvc files to update.",
+        "targets", nargs="+", help=".dvc files to update."
     ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",

dvc/compare.py

@@ -0,0 +1,321 @@
+from collections import abc
+from itertools import chain, repeat, zip_longest
+from operator import itemgetter
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    ItemsView,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    MutableSequence,
+    Sequence,
+    Tuple,
+    Union,
+    overload,
+)
+
+from funcy import reraise
+
+if TYPE_CHECKING:
+    from dvc.ui.table import CellT
+
+
+class Column(List["CellT"]):
+    pass
+
+
+def with_value(value, default):
+    return default if value is None else value
+
+
+class TabularData(MutableSequence[Sequence["CellT"]]):
+    def __init__(self, columns: Sequence[str], fill_value: str = ""):
+        self._columns: Dict[str, Column] = {name: Column() for name in columns}
+        self._keys: List[str] = list(columns)
+        self._fill_value = fill_value
+
+    @property
+    def columns(self) -> List[Column]:
+        return list(map(self.column, self.keys()))
+
+    def column(self, name: str) -> Column:
+        return self._columns[name]
+
+    def items(self) -> ItemsView[str, Column]:
+        projection = {k: self.column(k) for k in self.keys()}
+        return projection.items()
+
+    def keys(self) -> List[str]:
+        return self._keys
+
+    def _iter_col_row(
+        self, row: Sequence["CellT"]
+    ) -> Iterator[Tuple["CellT", Column]]:
+        for val, col in zip_longest(row, self.columns):
+            if col is None:
+                break
+            yield with_value(val, self._fill_value), col
+
+    def append(self, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.append(val)
+
+    def extend(self, values: Iterable[Sequence["CellT"]]) -> None:
+        for row in values:
+            self.append(row)
+
+    def insert(self, index: int, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.insert(index, val)
+
+    def __iter__(self) -> Iterator[List["CellT"]]:
+        return map(list, zip(*self.columns))
+
+    def __getattr__(self, item: str) -> Column:
+        with reraise(KeyError, AttributeError):
+            return self.column(item)
+
+    def __getitem__(self, item: Union[int, slice]):
+        func = itemgetter(item)
+        it = map(func, self.columns)
+        if isinstance(item, slice):
+            it = map(list, zip(*it))
+        return list(it)
+
+    @overload
+    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None:
+        ...
+
+    @overload
+    def __setitem__(
+        self, item: slice, value: Iterable[Sequence["CellT"]]
+    ) -> None:
+        ...
+
+    def __setitem__(self, item, value) -> None:
+        it = value
+        if isinstance(item, slice):
+            n = len(self.columns)
+            normalized_rows = (
+                chain(val, repeat(self._fill_value, n - len(val)))
+                for val in value
+            )
+            # we need to transpose those rows into columnar format
+            # as we work in terms of column-based arrays
+            it = zip(*normalized_rows)
+
+        for i, col in self._iter_col_row(it):
+            col[item] = i
+
+    def __delitem__(self, item: Union[int, slice]) -> None:
+        for col in self.columns:
+            del col[item]
+
+    def __len__(self) -> int:
+        return len(self.columns[0])
+
+    @property
+    def shape(self) -> Tuple[int, int]:
+        return len(self.columns), len(self)
+
+    def drop(self, *col_names: str) -> None:
+        for col_name in col_names:
+            self._keys.remove(col_name)
+            self._columns.pop(col_name)
+
+    def rename(self, from_col_name: str, to_col_name: str) -> None:
+        self._columns[to_col_name] = self._columns.pop(from_col_name)
+        self._keys[self._keys.index(from_col_name)] = to_col_name
+
+    def project(self, *col_names: str) -> None:
+        self.drop(*(set(self._keys) - set(col_names)))
+        self._keys = list(col_names)
+
+    def to_csv(self) -> str:
+        import csv
+        from io import StringIO
+
+        buff = StringIO()
+        writer = csv.writer(buff)
+        writer.writerow(self.keys())
+
+        for row in self:
+            writer.writerow(row)
+        return buff.getvalue()
+
+    def add_column(self, name: str) -> None:
+        self._columns[name] = Column([self._fill_value] * len(self))
+        self._keys.append(name)
+
+    def row_from_dict(self, d: Mapping[str, "CellT"]) -> None:
+        keys = self.keys()
+        for key in d:
+            if key not in keys:
+                self.add_column(key)
+
+        row: List["CellT"] = [
+            with_value(d.get(key), self._fill_value) for key in self.keys()
+        ]
+        self.append(row)
+
+    def render(self, **kwargs: Any):
+        from dvc.ui import ui
+
+        ui.table(self, headers=self.keys(), **kwargs)
+
+    def as_dict(
+        self, cols: Iterable[str] = None
+    ) -> Iterable[Dict[str, "CellT"]]:
+        keys = self.keys() if cols is None else set(cols)
+        return [
+            {k: self._columns[k][i] for k in keys} for i in range(len(self))
+        ]
+
+
+def _normalize_float(val: float, precision: int):
+    return f"{val:.{precision}g}"
+
+
+def _format_field(
+    val: Any, precision: int = None, round_digits: bool = False
+) -> str:
+    def _format(_val):
+        if isinstance(_val, float) and precision:
+            func = round if round_digits else _normalize_float
+            return func(_val, precision)
+        if isinstance(_val, abc.Mapping):
+            return {k: _format(v) for k, v in _val.items()}
+        if isinstance(_val, list):
+            return [_format(x) for x in _val]
+        return _val
+
+    return str(_format(val))
+
+
+def diff_table(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+) -> TabularData:
+    headers: List[str] = ["Path", title, "Old", "New", "Change"]
+    fill_value = "-"
+    td = TabularData(headers, fill_value=fill_value)
+
+    for fname, diff_in_file in diff.items():
+        for item, change in sorted(diff_in_file.items()):
+            old_value = with_value(change.get("old"), fill_value)
+            new_value = with_value(change.get("new"), fill_value)
+            diff_value = with_value(
+                change.get("diff", on_empty_diff), fill_value
+            )
+            td.append(
+                [
+                    fname,
+                    str(item),
+                    _format_field(old_value, precision, round_digits),
+                    _format_field(new_value, precision, round_digits),
+                    _format_field(diff_value, precision, round_digits),
+                ]
+            )
+
+    if no_path:
+        td.drop("Path")
+
+    if not show_changes:
+        td.drop("Change")
+
+    if not old:
+        td.drop("Old")
+        td.rename("New", "Value")
+
+    return td
+
+
+def show_diff(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+    markdown: bool = False,
+) -> None:
+    td = diff_table(
+        diff,
+        title=title,
+        old=old,
+        no_path=no_path,
+        show_changes=show_changes,
+        precision=precision,
+        round_digits=round_digits,
+        on_empty_diff=on_empty_diff,
+    )
+    td.render(markdown=markdown)
+
+
+def metrics_table(
+    metrics,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+):
+    from dvc.utils.diff import format_dict
+    from dvc.utils.flatten import flatten
+
+    td = TabularData(["Revision", "Path"], fill_value="-")
+
+    for branch, val in metrics.items():
+        for fname, metric in val.items():
+            row_data: Dict[str, str] = {"Revision": branch, "Path": fname}
+            flattened = (
+                flatten(format_dict(metric))
+                if isinstance(metric, dict)
+                else {"": metric}
+            )
+            row_data.update(
+                {
+                    k: _format_field(v, precision, round_digits)
+                    for k, v in flattened.items()
+                }
+            )
+            td.row_from_dict(row_data)
+
+    rev, path, *metrics_headers = td.keys()
+    td.project(rev, path, *sorted(metrics_headers))
+
+    if not any([all_branches, all_tags, all_commits]):
+        td.drop("Revision")
+
+    return td
+
+
+def show_metrics(
+    metrics,
+    markdown: bool = False,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+) -> None:
+    td = metrics_table(
+        metrics,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        all_commits=all_commits,
+        precision=precision,
+        round_digits=round_digits,
+    )
+    td.render(markdown=markdown)

dvc/config.py

@@ -63,7 +63,7 @@ class Config(dict):
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True, fs=None, config=None,
+        self, dvc_dir=None, validate=True, fs=None, config=None
     ):  # pylint: disable=super-init-not-called
         from dvc.fs.local import LocalFileSystem
 
@@ -79,7 +79,7 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.wfs = LocalFileSystem(None, {"url": self.dvc_dir})
+        self.wfs = LocalFileSystem(url=self.dvc_dir)
         self.fs = fs or self.wfs
 
         self.load(validate=validate, config=config)
@@ -154,7 +154,7 @@ class Config(dict):
         filename = self.files[level]
         fs = self._get_fs(level)
 
-        if fs.exists(filename, use_dvcignore=False):
+        if fs.exists(filename):
             with fs.open(filename) as fobj:
                 conf_obj = ConfigObj(fobj)
         else:

dvc/config_schema.py

@@ -145,6 +145,7 @@ SCHEMA = {
                     "session_token": str,
                     Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
+                    Optional("ssl_verify", default=True): Bool,
                     "sse": str,
                     "sse_kms_key_id": str,
                     "acl": str,
@@ -171,7 +172,7 @@ SCHEMA = {
                     "allow_agent": Bool,
                     **REMOTE_COMMON,
                 },
-                "hdfs": {"user": str, **REMOTE_COMMON},
+                "hdfs": {"user": str, "kerb_ticket": str, **REMOTE_COMMON},
                 "webhdfs": {
                     "hdfscli_config": str,
                     "webhdfs_token": str,
@@ -222,4 +223,5 @@ SCHEMA = {
         # enabled by default. It's of no use, kept for backward compatibility.
         Optional("parametrization", default=True): Bool
     },
+    "plots": {"html_template": str},
 }

dvc/daemon.py

@@ -33,7 +33,7 @@ def _spawn_windows(cmd, env):
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
     _popen(
-        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
+        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo
     ).communicate()
 
 

dvc/dagascii.py

@@ -3,52 +3,14 @@
 import logging
 import math
 import os
-import pydoc
-import sys
 
 from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import EdgeViewer, route_with_lines
 
-from dvc.env import DVC_PAGER
-from dvc.utils import format_link
-
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_PAGER = "less"
-DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
-    DEFAULT_PAGER
-)
-
-
-def make_pager(cmd):
-    def pager(text):
-        return pydoc.tempfilepager(pydoc.plain(text), cmd)
-
-    return pager
-
-
-def find_pager():
-    if not sys.stdout.isatty():
-        return pydoc.plainpager
-
-    env_pager = os.getenv(DVC_PAGER)
-    if env_pager:
-        return make_pager(env_pager)
-
-    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
-        return make_pager(DEFAULT_PAGER_FORMATTED)
-
-    logger.warning(
-        "Unable to find `less` in the PATH. Check out "
-        "{} for more info.".format(
-            format_link("https://man.dvc.org/pipeline/show")
-        )
-    )
-    return pydoc.plainpager
-
-
 class VertexViewer:
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.

dvc/data_cloud.py

@@ -48,9 +48,7 @@ class DataCloud:
 
         return get_remote(self.repo, name=name)
 
-    def push(
-        self, cache, jobs=None, remote=None, show_checksums=False,
-    ):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
@@ -70,9 +68,7 @@ class DataCloud:
             show_checksums=show_checksums,
         )
 
-    def pull(
-        self, cache, jobs=None, remote=None, show_checksums=False,
-    ):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
@@ -92,24 +88,6 @@ class DataCloud:
             show_checksums=show_checksums,
         )
 
-    def transfer(self, source, jobs=None, remote=None, command=None):
-        """Transfer data items in a cloud-agnostic way.
-
-        Args:
-            source (str): url for the source location.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to compare
-                cache to. By default remote from core.remote config option
-                is used.
-            command (str): the command which is benefitting from this function
-                (to be used for reporting better error messages).
-        """
-        from dvc.fs import get_cloud_fs
-
-        from_fs = get_cloud_fs(self.repo, url=source)
-        remote = self.get_remote(remote, command)
-        return remote.transfer(from_fs, from_fs.path_info, jobs=jobs)
-
     def status(
         self,
         cache,

dvc/dependency/__init__.py

@@ -1,58 +1,24 @@
 from collections import defaultdict
-from urllib.parse import urlparse
+from typing import Any, Mapping
 
-import dvc.output as output
-from dvc.dependency.azure import AzureDependency
-from dvc.dependency.gs import GSDependency
-from dvc.dependency.hdfs import HDFSDependency
-from dvc.dependency.http import HTTPDependency
-from dvc.dependency.https import HTTPSDependency
-from dvc.dependency.local import LocalDependency
-from dvc.dependency.param import ParamsDependency
-from dvc.dependency.s3 import S3Dependency
-from dvc.dependency.ssh import SSHDependency
-from dvc.dependency.webdav import WebDAVDependency
-from dvc.dependency.webdavs import WebDAVSDependency
-from dvc.dependency.webhdfs import WebHDFSDependency
-from dvc.output.base import BaseOutput
-from dvc.scheme import Schemes
+from dvc.output import ARTIFACT_SCHEMA, Output
 
-from ..fs import get_cloud_fs
+from .base import Dependency
+from .param import ParamsDependency
 from .repo import RepoDependency
 
-DEP_MAP = {
-    Schemes.LOCAL: LocalDependency,
-    Schemes.SSH: SSHDependency,
-    Schemes.S3: S3Dependency,
-    Schemes.AZURE: AzureDependency,
-    Schemes.GS: GSDependency,
-    Schemes.HDFS: HDFSDependency,
-    Schemes.HTTP: HTTPDependency,
-    Schemes.HTTPS: HTTPSDependency,
-    Schemes.WEBDAV: WebDAVDependency,
-    Schemes.WEBDAVS: WebDAVSDependency,
-    Schemes.WEBHDFS: WebHDFSDependency,
-}
-
-
 # NOTE: schema for dependencies is basically the same as for outputs, but
 # without output-specific entries like 'cache' (whether or not output is
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metrics file and how to parse it, see `-M` flag for `dvc run`).
-SCHEMA = output.SCHEMA.copy()
-del SCHEMA[BaseOutput.PARAM_CACHE]
-del SCHEMA[BaseOutput.PARAM_METRIC]
-del SCHEMA[BaseOutput.PARAM_DESC]
-SCHEMA.update(RepoDependency.REPO_SCHEMA)
-SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
+SCHEMA: Mapping[str, Any] = {
+    **ARTIFACT_SCHEMA,
+    **RepoDependency.REPO_SCHEMA,
+    **ParamsDependency.PARAM_SCHEMA,
+}
 
 
 def _get(stage, p, info):
-    parsed = urlparse(p) if p else None
-    if parsed and parsed.scheme == "remote":
-        fs = get_cloud_fs(stage.repo, name=parsed.netloc)
-        return DEP_MAP[fs.scheme](stage, p, info, fs=fs)
-
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)
         return RepoDependency(repo, stage, p, info)
@@ -61,14 +27,13 @@ def _get(stage, p, info):
         params = info.pop(ParamsDependency.PARAM_PARAMS)
         return ParamsDependency(stage, p, params)
 
-    dep_cls = DEP_MAP.get(parsed.scheme, LocalDependency)
-    return dep_cls(stage, p, info)
+    return Dependency(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(BaseOutput.PARAM_PATH, None)
+        p = d.pop(Output.PARAM_PATH, None)
         ret.append(_get(stage, p, d))
     return ret
 

dvc/dependency/azure.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.azure import AzureFileSystem
-
-
-class AzureDependency(BaseDependency, BaseOutput):
-    FS_CLS = AzureFileSystem

dvc/dependency/base.py

@@ -1,6 +1,7 @@
 from typing import Type
 
 from dvc.exceptions import DvcException
+from dvc.output import Output
 
 
 class DependencyDoesNotExistError(DvcException):
@@ -20,7 +21,7 @@ class DependencyIsStageFileError(DvcException):
         super().__init__(f"DVC file '{path}' cannot be a dependency.")
 
 
-class BaseDependency:
+class Dependency(Output):
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError  # type: Type[DvcException]

dvc/dependency/gs.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.gs import GSFileSystem
-
-
-class GSDependency(BaseDependency, BaseOutput):
-    FS_CLS = GSFileSystem

dvc/dependency/hdfs.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.hdfs import HDFSOutput
-
-
-class HDFSDependency(BaseDependency, HDFSOutput):
-    pass

dvc/dependency/http.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.http import HTTPFileSystem
-
-
-class HTTPDependency(BaseDependency, BaseOutput):
-    FS_CLS = HTTPFileSystem

dvc/dependency/https.py

@@ -1,6 +0,0 @@
-from ..fs.https import HTTPSFileSystem
-from .http import HTTPDependency
-
-
-class HTTPSDependency(HTTPDependency):
-    FS_CLS = HTTPSFileSystem

dvc/dependency/local.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.local import LocalOutput
-
-
-class LocalDependency(BaseDependency, LocalOutput):
-    pass

dvc/dependency/param.py

@@ -4,11 +4,12 @@ from collections import defaultdict
 import dpath.util
 from voluptuous import Any
 
-from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
 from dvc.hash_info import HashInfo
 from dvc.utils.serialize import LOADERS, ParseError
 
+from .base import Dependency
+
 
 class MissingParamsError(DvcException):
     pass
@@ -18,7 +19,7 @@ class BadParamFileError(DvcException):
     pass
 
 
-class ParamsDependency(LocalDependency):
+class ParamsDependency(Dependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
@@ -84,19 +85,34 @@ class ParamsDependency(LocalDependency):
     def status(self):
         return self.workspace_status()
 
-    def read_params(self):
+    def _read(self):
         if not self.exists:
             return {}
 
         suffix = self.path_info.suffix.lower()
         loader = LOADERS[suffix]
         try:
-            config = loader(self.path_info, fs=self.repo.fs)
+            return loader(self.path_info, fs=self.repo.fs)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
+    def read_params_d(self):
+        config = self._read()
+
+        ret = {}
+        for param in self.params:
+            dpath.util.merge(
+                ret,
+                dpath.util.search(config, param, separator="."),
+                separator=".",
+            )
+        return ret
+
+    def read_params(self):
+        config = self._read()
+
         ret = {}
         for param in self.params:
             try:
@@ -112,7 +128,7 @@ class ParamsDependency(LocalDependency):
         if missing_params:
             raise MissingParamsError(
                 "Parameters '{}' are missing from '{}'.".format(
-                    ", ".join(missing_params), self,
+                    ", ".join(missing_params), self
                 )
             )
 

dvc/dependency/repo.py

@@ -2,10 +2,10 @@ from voluptuous import Required
 
 from dvc.path_info import PathInfo
 
-from .local import LocalDependency
+from .base import Dependency
 
 
-class RepoDependency(LocalDependency):
+class RepoDependency(Dependency):
     PARAM_REPO = "repo"
     PARAM_URL = "url"
     PARAM_REV = "rev"
@@ -23,7 +23,7 @@ class RepoDependency(LocalDependency):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, fs, path):
+    def _parse_path(self, fs, path_info):
         return None
 
     @property
@@ -56,7 +56,6 @@ class RepoDependency(LocalDependency):
                 path_info,
                 repo.repo_fs,
                 self.repo.odb.local.fs.PARAM_CHECKSUM,
-                follow_subrepos=False,
             ).hash_info
 
     def workspace_status(self):
@@ -95,16 +94,18 @@ class RepoDependency(LocalDependency):
             except (NoOutputOrStageError, NoRemoteError):
                 pass
             obj = stage(
-                odb,
-                path_info,
-                repo.repo_fs,
-                odb.fs.PARAM_CHECKSUM,
-                jobs=jobs,
-                follow_subrepos=False,
+                odb, path_info, repo.repo_fs, odb.fs.PARAM_CHECKSUM, jobs=jobs
             )
             save(odb, obj, jobs=jobs)
 
-        checkout(to.path_info, to.fs, obj, odb)
+        checkout(
+            to.path_info,
+            to.fs,
+            obj,
+            odb,
+            dvcignore=None,
+            state=self.repo.state,
+        )
 
     def update(self, rev=None):
         if rev:

dvc/dependency/s3.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.s3 import S3Output
-
-
-class S3Dependency(BaseDependency, S3Output):
-    pass

dvc/dependency/ssh.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.ssh import SSHOutput
-
-
-class SSHDependency(BaseDependency, SSHOutput):
-    pass

dvc/dependency/webdav.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.webdav import WebDAVFileSystem
-
-
-class WebDAVDependency(BaseDependency, BaseOutput):
-    FS_CLS = WebDAVFileSystem

dvc/dependency/webdavs.py

@@ -1,6 +0,0 @@
-from ..fs.webdavs import WebDAVSFileSystem
-from .webdav import WebDAVDependency
-
-
-class WebDAVSDependency(WebDAVDependency):
-    FS_CLS = WebDAVSFileSystem

dvc/dependency/webhdfs.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.webhdfs import WebHDFSOutput
-
-
-class WebHDFSDependency(BaseDependency, WebHDFSOutput):
-    pass

dvc/dvcfile.py

@@ -37,8 +37,12 @@ PIPELINE_LOCK = "dvc.lock"
 
 
 class FileIsGitIgnored(DvcException):
-    def __init__(self, path):
-        super().__init__(f"'{path}' is git-ignored.")
+    def __init__(self, path, pipeline_file=False):
+        super().__init__(
+            "{}'{}' is git-ignored.".format(
+                "bad DVC file name " if pipeline_file else "", path
+            )
+        )
 
 
 class LockfileCorruptedError(DvcException):
@@ -66,7 +70,19 @@ def is_lock_file(path):
     return os.path.basename(path) == PIPELINE_LOCK
 
 
-def check_dvc_filename(path):
+def is_git_ignored(repo, path):
+    from dvc.fs.local import LocalFileSystem
+    from dvc.scm.base import NoSCMError
+
+    try:
+        return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(
+            path
+        )
+    except NoSCMError:
+        return False
+
+
+def check_dvcfile_path(repo, path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
             "bad DVC file name '{}'. DVC files should be named "
@@ -75,6 +91,9 @@ def check_dvc_filename(path):
             )
         )
 
+    if is_git_ignored(repo, path):
+        raise FileIsGitIgnored(relpath(path), True)
+
 
 class FileMixin:
     SCHEMA = None
@@ -105,18 +124,15 @@ class FileMixin:
         return relpath(self.path)
 
     def exists(self):
-        return self.repo.fs.exists(self.path)
+        is_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+        return self.repo.fs.exists(self.path) and not is_ignored
 
     def _is_git_ignored(self):
-        from dvc.fs.local import LocalFileSystem
-
-        return isinstance(
-            self.repo.fs, LocalFileSystem
-        ) and self.repo.scm.is_ignored(self.path)
+        return is_git_ignored(self.repo, self.path)
 
     def _verify_filename(self):
         if self.verify:
-            check_dvc_filename(self.path)
+            check_dvcfile_path(self.repo, self.path)
 
     def _check_gitignored(self):
         if self._is_git_ignored():
@@ -129,8 +145,10 @@ class FileMixin:
         # 3. path doesn't represent a regular file
         # 4. when the file is git ignored
         if not self.exists():
-            is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
-            raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
+            dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+            raise StageFileDoesNotExistError(
+                self.path, dvc_ignored=dvc_ignored
+            )
 
         self._verify_filename()
         if not self.repo.fs.isfile(self.path):
@@ -181,7 +199,7 @@ class SingleStageFile(FileMixin):
 
         assert not isinstance(stage, PipelineStage)
         if self.verify:
-            check_dvc_filename(self.path)
+            check_dvcfile_path(self.repo, self.path)
         logger.debug(
             "Saving information to '{file}'.".format(file=relpath(self.path))
         )
@@ -218,7 +236,7 @@ class PipelineFile(FileMixin):
 
         assert isinstance(stage, PipelineStage)
         if self.verify:
-            check_dvc_filename(self.path)
+            check_dvcfile_path(self.repo, self.path)
 
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)

dvc/exceptions.py

@@ -35,7 +35,7 @@ class OutputDuplicationError(DvcException):
             )
         else:
             msg = "output '{}' is already specified in stages:\n{}".format(
-                output, "\n".join(f"\t- {s.addressing}" for s in stages),
+                output, "\n".join(f"\t- {s.addressing}" for s in stages)
             )
         super().__init__(msg)
         self.stages = stages
@@ -283,7 +283,7 @@ class CheckoutError(DvcException):
         m = (
             "Checkout failed for following targets:\n{}\nIs your "
             "cache up to date?\n{}".format(
-                "\n".join(targets), error_link("missing-files"),
+                "\n".join(targets), error_link("missing-files")
             )
         )
         super().__init__(m)
@@ -380,7 +380,7 @@ class CacheLinkError(DvcException):
 
     def __init__(self, path_infos):
         msg = "No possible cache link types for '{}'. {}".format(
-            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK,
+            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK
         )
         super().__init__(msg)
         self.path_infos = path_infos

dvc/external_repo.py

@@ -36,10 +36,7 @@ def external_repo(
     rev = rev or "refs/remotes/origin/HEAD"
 
     cache_config = {
-        "cache": {
-            "dir": cache_dir or _get_cache_dir(url),
-            "type": cache_types,
-        }
+        "cache": {"dir": cache_dir or _get_cache_dir(url), "type": cache_types}
     }
 
     config = _get_remote_config(url) if os.path.isdir(url) else {}
@@ -144,7 +141,7 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    from distutils.dir_util import copy_tree
+    from shutil import copytree
 
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
@@ -156,7 +153,7 @@ def _cached_clone(url, rev, for_write=False):
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
     logger.debug("erepo: making a copy of %s clone", url)
-    copy_tree(clone_path, repo_path)
+    copytree(clone_path, repo_path)
 
     # Check out the specified revision
     if for_write:

dvc/fs/__init__.py

@@ -82,9 +82,28 @@ def get_cloud_fs(repo, **kwargs):
     from dvc.config import ConfigError
     from dvc.config_schema import SCHEMA, Invalid
 
-    remote_conf = get_fs_config(repo.config, **kwargs)
+    repo_config = repo.config if repo else {}
+    core_config = repo_config.get("core", {})
+
+    remote_conf = get_fs_config(repo_config, **kwargs)
     try:
         remote_conf = SCHEMA["remote"][str](remote_conf)
     except Invalid as exc:
         raise ConfigError(str(exc)) from None
-    return get_fs_cls(remote_conf)(repo, remote_conf)
+
+    if "jobs" not in remote_conf:
+        jobs = core_config.get("jobs")
+        if jobs:
+            remote_conf["jobs"] = jobs
+
+    if "checksum_jobs" not in remote_conf:
+        checksum_jobs = core_config.get("checksum_jobs")
+        if checksum_jobs:
+            remote_conf["checksum_jobs"] = checksum_jobs
+
+    cls = get_fs_cls(remote_conf)
+
+    if cls == GDriveFileSystem and repo:
+        remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
+
+    return cls, remote_conf

dvc/fs/_metadata.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, List
 
-from dvc.output import BaseOutput
+from dvc.output import Output
 from dvc.path_info import PathInfo
 
 if TYPE_CHECKING:
@@ -33,7 +33,7 @@ class Metadata:
     # optional fields
     isdir: bool = False  # is it a directory?
     is_exec: bool = False  # is it an executable?
-    outs: List[BaseOutput] = field(default_factory=list)  # list of outputs
+    outs: List[Output] = field(default_factory=list)  # list of outputs
 
     def __post_init__(self):
         self.output_exists = bool(self.outs)

dvc/fs/azure.py

@@ -1,6 +1,9 @@
+import asyncio
 import logging
 import os
+import sys
 import threading
+from contextlib import contextmanager
 
 from funcy import cached_property, wrap_prop
 
@@ -9,7 +12,7 @@ from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 from dvc.utils import format_link
 
-from .fsspec_wrapper import FSSpecWrapper
+from .fsspec_wrapper import ObjectFSWrapper
 
 logger = logging.getLogger(__name__)
 _DEFAULT_CREDS_STEPS = (
@@ -19,11 +22,53 @@ _DEFAULT_CREDS_STEPS = (
 )
 
 
+@contextmanager
+def _temp_event_loop():
+    """When trying to initalize azure filesystem instances
+    with DefaultCredentials, the authentication process requires
+    to have an access to a separate event loop. The normal calls
+    are run in a separate loop managed by the fsspec, but the
+    DefaultCredentials assumes that the callee is managing their
+    own event loop. This function checks whether is there any
+    event loop set, and if not it creates a temporary event loop
+    and set it. After the context is finalized, it restores the
+    original event loop back (if there is any)."""
+
+    try:
+        original_loop = asyncio.get_event_loop()
+        original_policy = asyncio.get_event_loop_policy()
+    except RuntimeError:
+        original_loop = None
+        original_policy = None
+
+    # From 3.8>= and onwards, asyncio changed the default
+    # loop policy for windows to use proactor loops instead
+    # of selector based ones. Due to that, proxied connections
+    # doesn't work with the aiohttp and this is most likely an
+    # upstream bug that needs to be solved outside of DVC. Until
+    # such issue is resolved, we need to manage this;
+    # https://github.com/aio-libs/aiohttp/issues/4536
+    if sys.version_info >= (3, 8) and os.name == "nt":
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+    loop = original_loop or asyncio.new_event_loop()
+
+    try:
+        asyncio.set_event_loop(loop)
+        yield
+    finally:
+        if original_loop is None:
+            loop.close()
+        asyncio.set_event_loop(original_loop)
+        asyncio.set_event_loop_policy(original_policy)
+
+
 class AzureAuthError(DvcException):
     pass
 
 
-class AzureFileSystem(FSSpecWrapper):
+# pylint:disable=abstract-method
+class AzureFileSystem(ObjectFSWrapper):
     scheme = Schemes.AZURE
     PATH_CLS = CloudURLInfo
     PARAM_CHECKSUM = "etag"
@@ -34,8 +79,8 @@ class AzureFileSystem(FSSpecWrapper):
         "azure-identity": "azure.identity",
     }
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
         url = config.get("url")
         self.path_info = self.PATH_CLS(url)
@@ -48,9 +93,12 @@ class AzureFileSystem(FSSpecWrapper):
         self.path_info = self.PATH_CLS(url)
         self.bucket = self.path_info.bucket
 
-    def _prepare_credentials(self, config):
+    def _prepare_credentials(self, **config):
         from azure.identity.aio import DefaultAzureCredential
 
+        # Disable spam from failed cred types for DefaultAzureCredential
+        logging.getLogger("azure.identity.aio").setLevel(logging.ERROR)
+
         login_info = {}
         login_info["connection_string"] = config.get(
             "connection_string",
@@ -69,8 +117,12 @@ class AzureFileSystem(FSSpecWrapper):
         login_info["client_id"] = config.get("client_id")
         login_info["client_secret"] = config.get("client_secret")
 
-        if not any(login_info.values()):
-            login_info["credentials"] = DefaultAzureCredential()
+        if not any(
+            value for key, value in login_info.items() if key != "account_name"
+        ):
+            login_info["credential"] = DefaultAzureCredential(
+                exclude_interactive_browser_credential=False
+            )
 
         for login_method, required_keys in [  # noqa
             ("connection string", ["connection_string"]),
@@ -80,8 +132,11 @@ class AzureFileSystem(FSSpecWrapper):
             ),
             ("account key", ["account_name", "account_key"]),
             ("SAS token", ["account_name", "sas_token"]),
+            (
+                f"default credentials ({_DEFAULT_CREDS_STEPS})",
+                ["account_name", "credential"],
+            ),
             ("anonymous login", ["account_name"]),
-            (f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
         ]:
             if all(login_info.get(key) is not None for key in required_keys):
                 break
@@ -110,11 +165,12 @@ class AzureFileSystem(FSSpecWrapper):
         from azure.core.exceptions import AzureError
 
         try:
-            file_system = AzureBlobFileSystem(**self.fs_args)
-            if self.bucket not in [
-                container.rstrip("/") for container in file_system.ls("/")
-            ]:
-                file_system.mkdir(self.bucket)
+            with _temp_event_loop():
+                file_system = AzureBlobFileSystem(**self.fs_args)
+                if self.bucket not in [
+                    container.rstrip("/") for container in file_system.ls("/")
+                ]:
+                    file_system.mkdir(self.bucket)
         except (ValueError, AzureError) as e:
             raise AzureAuthError(
                 f"Authentication to Azure Blob Storage via {self.login_method}"
@@ -123,3 +179,9 @@ class AzureFileSystem(FSSpecWrapper):
             ) from e
 
         return file_system
+
+    def open(
+        self, path_info, mode="r", **kwargs
+    ):  # pylint: disable=arguments-differ
+        with _temp_event_loop():
+            return self.fs.open(self._with_bucket(path_info), mode=mode)

dvc/fs/base.py

@@ -1,11 +1,9 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
+from functools import partial, partialmethod
 from multiprocessing import cpu_count
 from typing import Any, ClassVar, Dict, FrozenSet, Optional
 
-from funcy import cached_property
-
 from dvc.exceptions import DvcException
 from dvc.path_info import URLInfo
 from dvc.progress import Tqdm
@@ -38,7 +36,7 @@ class BaseFileSystem:
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
     PATH_CLS = URLInfo  # type: Any
-    JOBS = 4 * cpu_count()
+    _JOBS = 4 * cpu_count()
 
     CHECKSUM_DIR_SUFFIX = ".dir"
     HASH_JOBS = max(1, min(4, cpu_count() // 2))
@@ -54,29 +52,13 @@ class BaseFileSystem:
     PARAM_CHECKSUM: ClassVar[Optional[str]] = None
     DETAIL_FIELDS: FrozenSet[str] = frozenset()
 
-    def __init__(self, repo, config):
-        self.repo = repo
-        self.config = config
-
-        self._check_requires()
+    def __init__(self, **kwargs):
+        self._check_requires(**kwargs)
 
         self.path_info = None
 
-    @cached_property
-    def jobs(self):
-        return (
-            self.config.get("jobs")
-            or (self.repo and self.repo.config["core"].get("jobs"))
-            or self.JOBS
-        )
-
-    @cached_property
-    def hash_jobs(self):
-        return (
-            self.config.get("checksum_jobs")
-            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
-            or self.HASH_JOBS
-        )
+        self.jobs = kwargs.get("jobs") or self._JOBS
+        self.hash_jobs = kwargs.get("checksum_jobs") or self.HASH_JOBS
 
     @classmethod
     def get_missing_deps(cls):
@@ -91,7 +73,7 @@ class BaseFileSystem:
 
         return missing
 
-    def _check_requires(self):
+    def _check_requires(self, **kwargs):
         from ..scheme import Schemes
         from ..utils import format_link
         from ..utils.pkg import PKG
@@ -100,7 +82,7 @@ class BaseFileSystem:
         if not missing:
             return
 
-        url = self.config.get("url", f"{self.scheme}://")
+        url = kwargs.get("url", f"{self.scheme}://")
 
         scheme = self.scheme
         if scheme == Schemes.WEBDAVS:
@@ -139,7 +121,7 @@ class BaseFileSystem:
 
         raise RemoteActionNotImplemented("open", self.scheme)
 
-    def exists(self, path_info, use_dvcignore=True) -> bool:
+    def exists(self, path_info) -> bool:
         raise NotImplementedError
 
     # pylint: disable=unused-argument
@@ -166,6 +148,10 @@ class BaseFileSystem:
         """Check if this file is an independent copy."""
         return False  # We can't be sure by default
 
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        """Return a generator with (root, dirs, files)."""
+        raise NotImplementedError
+
     def walk_files(self, path_info, **kwargs):
         """Return a generator with `PathInfo`s to all the files.
 
@@ -175,9 +161,12 @@ class BaseFileSystem:
         """
         raise NotImplementedError
 
-    def ls(self, path_info, detail=False, **kwargs):
+    def ls(self, path_info, detail=False):
         raise RemoteActionNotImplemented("ls", self.scheme)
 
+    def find(self, path_info, detail=False):
+        raise RemoteActionNotImplemented("find", self.scheme)
+
     def is_empty(self, path_info):
         return False
 
@@ -219,9 +208,7 @@ class BaseFileSystem:
             return False
         return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
-    def upload(
-        self, from_info, to_info, name=None, no_progress_bar=False,
-    ):
+    def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):
             raise RemoteActionNotImplemented("upload", self.scheme)
 
@@ -258,6 +245,7 @@ class BaseFileSystem:
         name=None,
         no_progress_bar=False,
         jobs=None,
+        _only_file=False,
         **kwargs,
     ):
         if not hasattr(self, "_download"):
@@ -273,14 +261,16 @@ class BaseFileSystem:
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if not _only_file and self.isdir(from_info):
             return self._download_dir(
-                from_info, to_info, name, no_progress_bar, jobs, **kwargs,
+                from_info, to_info, name, no_progress_bar, jobs, **kwargs
             )
-        return self._download_file(from_info, to_info, name, no_progress_bar,)
+        return self._download_file(from_info, to_info, name, no_progress_bar)
+
+    download_file = partialmethod(download, _only_file=True)
 
     def _download_dir(
-        self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
+        self, from_info, to_info, name, no_progress_bar, jobs, **kwargs
     ):
         from_infos = list(self.walk_files(from_info, **kwargs))
         if not from_infos:
@@ -297,7 +287,7 @@ class BaseFileSystem:
             disable=no_progress_bar,
         ) as pbar:
             download_files = pbar.wrap_fn(
-                partial(self._download_file, name=name, no_progress_bar=True,)
+                partial(self._download_file, name=name, no_progress_bar=True)
             )
             max_workers = jobs or self.jobs
             with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -321,9 +311,7 @@ class BaseFileSystem:
                             entry.cancel()
                         raise exc
 
-    def _download_file(
-        self, from_info, to_info, name, no_progress_bar,
-    ):
+    def _download_file(self, from_info, to_info, name, no_progress_bar):
         makedirs(to_info.parent, exist_ok=True)
 
         logger.debug("Downloading '%s' to '%s'", from_info, to_info)

dvc/fs/dvc.py

@@ -10,7 +10,7 @@ from ._metadata import Metadata
 from .base import BaseFileSystem
 
 if typing.TYPE_CHECKING:
-    from dvc.output.base import BaseOutput
+    from dvc.output import Output
 
 
 logger = logging.getLogger(__name__)
@@ -26,8 +26,9 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 
-    def __init__(self, repo):
-        super().__init__(repo, {"url": repo.root_dir})
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.repo = kwargs["repo"]
 
     def _find_outs(self, path, *args, **kwargs):
         outs = self.repo.find_outs_by_path(path, *args, **kwargs)
@@ -42,7 +43,7 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         return outs
 
     def _get_granular_hash(
-        self, path_info: PathInfo, out: "BaseOutput", remote=None
+        self, path_info: PathInfo, out: "Output", remote=None
     ):
         assert isinstance(path_info, PathInfo)
         # NOTE: use string paths here for performance reasons
@@ -67,13 +68,17 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             raise IsADirectoryError
 
         out = outs[0]
+
+        if not out.hash_info:
+            raise FileNotFoundError
+
         if out.changed_cache(filter_info=path):
             from dvc.config import NoRemoteError
 
             try:
                 remote_obj = self.repo.cloud.get_remote(remote)
-            except NoRemoteError:
-                raise FileNotFoundError
+            except NoRemoteError as exc:
+                raise FileNotFoundError from exc
             if out.is_dir_checksum:
                 checksum = self._get_granular_hash(path, out).value
             else:

dvc/fs/fsspec_wrapper.py

@@ -2,6 +2,8 @@ import os
 import shutil
 from functools import lru_cache
 
+from funcy import cached_property
+
 from dvc.progress import Tqdm
 
 from .base import BaseFileSystem
@@ -9,11 +11,12 @@ from .base import BaseFileSystem
 
 # pylint: disable=no-member
 class FSSpecWrapper(BaseFileSystem):
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
         self.fs_args = {"skip_instance_cache": True}
-        self.fs_args.update(self._prepare_credentials(config))
+        self.fs_args.update(self._prepare_credentials(**kwargs))
 
+    @cached_property
     def fs(self):
         raise NotImplementedError
 
@@ -33,15 +36,13 @@ class FSSpecWrapper(BaseFileSystem):
 
         return path or bucket
 
-    def _strip_buckets(self, entries, detail, prefix=None):
+    def _strip_buckets(self, entries, detail=False):
         for entry in entries:
             if detail:
                 entry = self._entry_hook(entry.copy())
                 entry["name"] = self._strip_bucket(entry["name"])
             else:
-                entry = self._strip_bucket(
-                    f"{prefix}/{entry}" if prefix else entry
-                )
+                entry = self._strip_bucket(entry)
             yield entry
 
     def _entry_hook(self, entry):
@@ -49,33 +50,28 @@ class FSSpecWrapper(BaseFileSystem):
         entries within info() and ls(detail=True) calls"""
         return entry
 
-    def _prepare_credentials(self, config):  # pylint: disable=unused-argument
+    def _prepare_credentials(
+        self, **config
+    ):  # pylint: disable=unused-argument
         """Prepare the arguments for authentication to the
         host filesystem"""
         return {}
 
-    def isdir(self, path_info):
-        if not self.exists(path_info):
-            return False
+    def _isdir(self, path_info):
+        return self.fs.isdir(self._with_bucket(path_info))
 
-        # Directory in object storages are interpreted differently
-        # among different fsspec providers, so this logic is a temporary
-        # measure for us to adapt as of now. It checks whether it is a
-        # directory (as in a prefix with contents) or whether it is an empty
-        # file where it's name ends with a forward slash
-        entry = self.info(path_info)
-        return entry["type"] == "directory" or (
-            entry["size"] == 0
-            and entry["type"] == "file"
-            and entry["name"].endswith("/")
-        )
+    def isdir(self, path_info):
+        try:
+            return self._isdir(path_info)
+        except FileNotFoundError:
+            return False
 
     def isfile(self, path_info):
-        if not self.exists(path_info):
+        try:
+            return not self._isdir(path_info)
+        except FileNotFoundError:
             return False
 
-        return not self.isdir(path_info)
-
     def is_empty(self, path_info):
         entry = self.info(path_info)
         if entry["type"] == "directory":
@@ -90,27 +86,24 @@ class FSSpecWrapper(BaseFileSystem):
     def copy(self, from_info, to_info):
         self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
 
-    def exists(self, path_info, use_dvcignore=False):
+    def exists(self, path_info) -> bool:
         return self.fs.exists(self._with_bucket(path_info))
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        if self.isdir(path_info) and self.is_empty(path_info):
-            return None
-
+    def ls(self, path_info, detail=False):
         path = self._with_bucket(path_info)
-        if recursive:
-            for root, _, files in self.fs.walk(path, detail=detail):
-                if detail:
-                    files = files.values()
-                yield from self._strip_buckets(files, detail, prefix=root)
-            return None
+        files = self.fs.ls(path, detail=detail)
+        yield from self._strip_buckets(files, detail=detail)
 
-        yield from self._strip_buckets(self.ls(path, detail=detail), detail)
+    def find(self, path_info, detail=False):
+        path = self._with_bucket(path_info)
+        files = self.fs.find(path, detail=detail)
+        if detail:
+            files = files.values()
+
+        yield from self._strip_buckets(files, detail=detail)
 
     def walk_files(self, path_info, **kwargs):
-        for file in self.ls(path_info, recursive=True):
+        for file in self.find(path_info):
             yield path_info.replace(path=file)
 
     def remove(self, path_info):
@@ -156,3 +149,36 @@ class FSSpecWrapper(BaseFileSystem):
             ) as wrapped:
                 with open(to_file, "wb") as fdest:
                     shutil.copyfileobj(wrapped, fdest, length=fobj.blocksize)
+
+
+# pylint: disable=abstract-method
+class ObjectFSWrapper(FSSpecWrapper):
+    def _isdir(self, path_info):
+        # Directory in object storages are interpreted differently
+        # among different fsspec providers, so this logic is a temporary
+        # measure for us to adapt as of now. It checks whether it is a
+        # directory (as in a prefix with contents) or whether it is an empty
+        # file where it's name ends with a forward slash
+
+        entry = self.info(path_info)
+        return entry["type"] == "directory" or (
+            entry["size"] == 0
+            and entry["type"] == "file"
+            and entry["name"].endswith("/")
+        )
+
+    def find(self, path_info, detail=False):
+        path = self._with_bucket(path_info)
+        files = self.fs.find(path, detail=detail)
+        if detail:
+            files = files.values()
+
+        # When calling find() on a file, it returns the same file in a list.
+        # For object-based storages, the same behavior applies to empty
+        # directories since they are represented as files. This condition
+        # checks whether we should yield an empty list (if it is an empty
+        # directory) or just yield the file itself.
+        if len(files) == 1 and files[0] == path and self.isdir(path_info):
+            return None
+
+        yield from self._strip_buckets(files, detail=detail)

dvc/fs/gdrive.py

@@ -85,7 +85,7 @@ class GDriveURLInfo(CloudURLInfo):
         self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
 
 
-class GDriveFileSystem(BaseFileSystem):
+class GDriveFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     scheme = Schemes.GDRIVE
     PATH_CLS = GDriveURLInfo
     PARAM_CHECKSUM = "checksum"
@@ -100,8 +100,8 @@ class GDriveFileSystem(BaseFileSystem):
     DEFAULT_GDRIVE_CLIENT_ID = "710796635688-iivsgbgsb6uv1fap6635dhvuei09o66c.apps.googleusercontent.com"  # noqa: E501
     DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
         self.path_info = self.PATH_CLS(config["url"])
 
@@ -126,17 +126,19 @@ class GDriveFileSystem(BaseFileSystem):
         self._client_id = config.get("gdrive_client_id")
         self._client_secret = config.get("gdrive_client_secret")
         self._validate_config()
+
+        tmp_dir = config["gdrive_credentials_tmp_dir"]
+        assert tmp_dir
+
         self._gdrive_service_credentials_path = tmp_fname(
-            os.path.join(self.repo.tmp_dir, "")
+            os.path.join(tmp_dir, "")
         )
         self._gdrive_user_credentials_path = (
-            tmp_fname(os.path.join(self.repo.tmp_dir, ""))
+            tmp_fname(os.path.join(tmp_dir, ""))
             if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
             else config.get(
                 "gdrive_user_credentials_file",
-                os.path.join(
-                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE,
-                ),
+                os.path.join(tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE),
             )
         )
 
@@ -517,7 +519,7 @@ class GDriveFileSystem(BaseFileSystem):
         assert not create
         raise FileMissingError(path_info, hint)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         try:
             self._get_item_id(path_info)
         except FileMissingError:
@@ -532,7 +534,7 @@ class GDriveFileSystem(BaseFileSystem):
         query = f"({query}) and trashed=false"
         return self._gdrive_list(query)
 
-    def _ls_recursive(self, path_info, detail=False):
+    def find(self, path_info, detail=False):
         root_path = path_info.path
         seen_paths = set()
 
@@ -569,13 +571,7 @@ class GDriveFileSystem(BaseFileSystem):
                 else:
                     yield item_path
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        if recursive:
-            yield from self._ls_recursive(path_info, detail=detail)
-            return None
-
+    def ls(self, path_info, detail=False):
         cached = path_info.path in self._ids_cache["dirs"]
         if cached:
             dir_ids = self._ids_cache["dirs"][path_info.path]
@@ -605,8 +601,8 @@ class GDriveFileSystem(BaseFileSystem):
             self._cache_path_id(root_path, *dir_ids)
 
     def walk_files(self, path_info, **kwargs):
-        for filename in self.ls(path_info, recursive=True):
-            yield path_info.replace(path=filename)
+        for file in self.find(path_info):
+            yield path_info.replace(path=file)
 
     def remove(self, path_info):
         item_id = self._get_item_id(path_info)

dvc/fs/git.py

@@ -1,8 +1,6 @@
 import errno
 import os
 
-from funcy import cached_property
-
 from dvc.utils import is_exec, relpath
 
 from .base import BaseFileSystem
@@ -11,43 +9,29 @@ from .base import BaseFileSystem
 class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
-    def __init__(
-        self, root_dir, trie, use_dvcignore=False, dvcignore_root=None
-    ):
-        super().__init__(None, {})
-        self._fs_root = root_dir
+    scheme = "local"
+
+    def __init__(self, root_dir, trie):
+        super().__init__()
+        self._root = root_dir
         self.trie = trie
-        self.use_dvcignore = use_dvcignore
-        self.dvcignore_root = dvcignore_root
 
     @property
     def rev(self):
         return self.trie.rev
 
-    @property
-    def fs_root(self):
-        return self._fs_root
-
     def _get_key(self, path):
         if isinstance(path, str):
             if not os.path.isabs(path):
                 relparts = path.split(os.sep)
             else:
-                relparts = relpath(path, self.fs_root).split(os.sep)
+                relparts = relpath(path, self._root).split(os.sep)
         else:
-            relparts = path.relative_to(self.fs_root).parts
+            relparts = path.relative_to(self._root).parts
         if relparts == ["."]:
             return ()
         return tuple(relparts)
 
-    @cached_property
-    def dvcignore(self):
-        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
-
-        root = self.dvcignore_root or self.fs_root
-        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
-        return cls(self, root)
-
     def open(
         self, path, mode="r", encoding=None
     ):  # pylint: disable=arguments-differ
@@ -66,50 +50,27 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 errno.EISDIR, os.strerror(errno.EISDIR), path
             ) from exc
 
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        def _is_ignored(path):
-            return self.dvcignore.is_ignored_file(
-                path
-            ) or self.dvcignore.is_ignored_dir(path)
-
-        if use_dvcignore and _is_ignored(path):
-            return False
-
-        key = self._get_key(path)
+    def exists(self, path_info) -> bool:
+        key = self._get_key(path_info)
         return self.trie.exists(key)
 
-    def isdir(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        if use_dvcignore and self.dvcignore.is_ignored_dir(path):
-            return False
-        key = self._get_key(path)
+    def isdir(self, path_info) -> bool:
+        key = self._get_key(path_info)
         return self.trie.isdir(key)
 
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        if self.dvcignore.is_ignored_file(path):
-            return False
-        key = self._get_key(path)
+    def isfile(self, path_info) -> bool:
+        key = self._get_key(path_info)
         return self.trie.isfile(key)
 
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        use_dvcignore=True,
-        ignore_subrepos=True,
-    ):
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
         - no support for symlinks
         """
-        if not self.isdir(top, use_dvcignore=use_dvcignore):
+        if not self.isdir(top):
             if onerror:
-                if self.exists(top, use_dvcignore=use_dvcignore):
+                if self.exists(top):
                     exc = NotADirectoryError(
                         errno.ENOTDIR, os.strerror(errno.ENOTDIR), top
                     )
@@ -123,13 +84,9 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         key = self._get_key(top)
         for prefix, dirs, files in self.trie.walk(key, topdown=topdown):
             if prefix:
-                root = os.path.join(self.fs_root, os.sep.join(prefix))
+                root = os.path.join(self._root, os.sep.join(prefix))
             else:
-                root = self.fs_root
-            if use_dvcignore:
-                dirs[:], files[:] = self.dvcignore(
-                    root, dirs, files, ignore_subrepos=ignore_subrepos,
-                )
+                root = self._root
             yield root, dirs, files
 
     def isexec(self, path_info):
@@ -158,11 +115,8 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 errno.ENOENT, os.strerror(errno.ENOENT), path
             )
 
-    def walk_files(self, top):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top):
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info, **kwargs):
             for file in files:
                 # NOTE: os.path.join is ~5.5 times slower
                 yield f"{root}{os.sep}{file}"
-
-    def _reset(self):
-        return self.__dict__.pop("dvcignore", None)

dvc/fs/gs.py

@@ -6,23 +6,24 @@ from funcy import cached_property, wrap_prop
 from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 
-from .fsspec_wrapper import FSSpecWrapper
+from .fsspec_wrapper import ObjectFSWrapper
 
 
-class GSFileSystem(FSSpecWrapper):
+# pylint:disable=abstract-method
+class GSFileSystem(ObjectFSWrapper):
     scheme = Schemes.GS
     PATH_CLS = CloudURLInfo
     REQUIRES = {"gcsfs": "gcsfs"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
         url = config.get("url", "gs://")
         self.path_info = self.PATH_CLS(url)
 
-    def _prepare_credentials(self, config):
+    def _prepare_credentials(self, **config):
         login_info = {"consistency": None}
         login_info["project"] = config.get("projectname")
         login_info["token"] = config.get("credentialpath")

dvc/fs/hdfs.py

@@ -65,8 +65,8 @@ class HDFSFileSystem(BaseFileSystem):
     PARAM_CHECKSUM = "checksum"
     TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
         self.path_info = None
         url = config.get("url")
@@ -84,8 +84,9 @@ class HDFSFileSystem(BaseFileSystem):
             path=parsed.path,
         )
 
-    @staticmethod
-    def hdfs(path_info):
+        self.kerb_ticket = config.get("kerb_ticket")
+
+    def hdfs(self, path_info):
         import pyarrow.fs
 
         # NOTE: HadoopFileSystem is not meant to be closed by us and doesn't
@@ -100,6 +101,7 @@ class HDFSFileSystem(BaseFileSystem):
             path_info.host,
             path_info.port,
             user=path_info.user,
+            kerb_ticket=self.kerb_ticket,
         )
 
     @contextmanager
@@ -121,7 +123,7 @@ class HDFSFileSystem(BaseFileSystem):
                 raise FileNotFoundError(*e.args)
             raise
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         assert not isinstance(path_info, list)
         assert path_info.scheme == "hdfs"
         with self.hdfs(path_info) as hdfs:
@@ -157,15 +159,15 @@ class HDFSFileSystem(BaseFileSystem):
         if not topdown:
             yield root, dirs, nondirs
 
-    def walk(self, path_info, **kwargs):
-        if not self.isdir(path_info):
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        if not self.isdir(top):
             return
 
-        with self.hdfs(path_info) as hdfs:
+        with self.hdfs(top) as hdfs:
             for root, dnames, fnames in self._walk(
-                hdfs, path_info.path, **kwargs
+                hdfs, top.path, topdown=topdown
             ):
-                yield path_info.replace(path=root), dnames, fnames
+                yield top.replace(path=root), dnames, fnames
 
     def walk_files(self, path_info, **kwargs):
         for root, _, fnames in self.walk(path_info):

dvc/fs/http.py

@@ -5,7 +5,7 @@ from typing import Optional
 
 from funcy import cached_property, memoize, wrap_prop, wrap_with
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.exceptions import DvcException, HTTPError
 from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
@@ -36,8 +36,8 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     REQUEST_TIMEOUT = 60
     CHUNK_SIZE = 2 ** 16
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
         url = config.get("url")
         if url:
@@ -138,7 +138,7 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         return response
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         res = self._head(path_info.url)
         if res.status_code == 404:
             return False

dvc/fs/local.py

@@ -2,8 +2,6 @@ import logging
 import os
 import stat
 
-from funcy import cached_property
-
 from dvc.path_info import PathInfo
 from dvc.scheme import Schemes
 from dvc.system import System
@@ -22,70 +20,34 @@ class LocalFileSystem(BaseFileSystem):
     PARAM_PATH = "path"
     TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config, use_dvcignore=False, dvcignore_root=None):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        from fsspec.implementations.local import LocalFileSystem as LocalFS
+
+        super().__init__(**config)
+        self.fs = LocalFS()
         url = config.get("url")
         self.path_info = self.PATH_CLS(url) if url else None
-        self.use_dvcignore = use_dvcignore
-        self.dvcignore_root = dvcignore_root
-
-    @property
-    def fs_root(self):
-        return self.config.get("url")
-
-    @cached_property
-    def dvcignore(self):
-        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
-
-        root = self.dvcignore_root or self.fs_root
-        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
-        return cls(self, root)
 
     @staticmethod
     def open(path_info, mode="r", encoding=None, **kwargs):
         return open(path_info, mode=mode, encoding=encoding)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         assert isinstance(path_info, str) or path_info.scheme == "local"
-        if self.repo:
-            ret = os.path.lexists(path_info)
-        else:
-            ret = os.path.exists(path_info)
-        if not ret:
-            return False
-        if not use_dvcignore:
-            return True
+        return self.fs.exists(path_info)
 
-        return not self.dvcignore.is_ignored_file(
-            path_info
-        ) and not self.dvcignore.is_ignored_dir(path_info)
+    def isfile(self, path_info) -> bool:
+        return os.path.isfile(path_info)
 
-    def isfile(self, path_info):
-        if not os.path.isfile(path_info):
-            return False
-
-        return not self.dvcignore.is_ignored_file(path_info)
-
-    def isdir(
-        self, path_info, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        if not os.path.isdir(path_info):
-            return False
-        return not (use_dvcignore and self.dvcignore.is_ignored_dir(path_info))
+    def isdir(self, path_info) -> bool:
+        return os.path.isdir(path_info)
 
     def iscopy(self, path_info):
         return not (
             System.is_symlink(path_info) or System.is_hardlink(path_info)
         )
 
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        use_dvcignore=True,
-        ignore_subrepos=True,
-    ):
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Directory fs generator.
 
         See `os.walk` for the docs. Differences:
@@ -94,14 +56,6 @@ class LocalFileSystem(BaseFileSystem):
         for root, dirs, files in os.walk(
             top, topdown=topdown, onerror=onerror
         ):
-            if use_dvcignore:
-                dirs[:], files[:] = self.dvcignore(
-                    os.path.abspath(root),
-                    dirs,
-                    files,
-                    ignore_subrepos=ignore_subrepos,
-                )
-
             yield os.path.normpath(root), dirs, files
 
     def walk_files(self, path_info, **kwargs):
@@ -133,9 +87,6 @@ class LocalFileSystem(BaseFileSystem):
         return is_exec(mode)
 
     def stat(self, path):
-        if self.dvcignore.is_ignored(path):
-            raise FileNotFoundError
-
         return os.stat(path)
 
     def move(self, from_info, to_info):
@@ -210,7 +161,7 @@ class LocalFileSystem(BaseFileSystem):
         }
 
     def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs,
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
         makedirs(to_info.parent, exist_ok=True)
 
@@ -227,6 +178,3 @@ class LocalFileSystem(BaseFileSystem):
         copyfile(
             from_info, to_file, no_progress_bar=no_progress_bar, name=name
         )
-
-    def _reset(self):
-        return self.__dict__.pop("dvcignore", None)

dvc/fs/memory.py

@@ -5,14 +5,14 @@ class MemoryFileSystem(BaseFileSystem):
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 
-    def __init__(self, repo, config):
+    def __init__(self, **kwargs):
         from fsspec.implementations.memory import MemoryFileSystem as MemFS
 
-        super().__init__(repo, config)
+        super().__init__(**kwargs)
 
         self.fs = MemFS()
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         return self.fs.exists(path_info.fspath)
 
     def open(self, path_info, mode="r", encoding=None, **kwargs):

dvc/fs/oss.py

@@ -39,8 +39,8 @@ class OSSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 100
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
         url = config.get("url")
         self.path_info = self.PATH_CLS(url) if url else None
@@ -88,7 +88,7 @@ class OSSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         return self.oss_service.sign_url("GET", path_info.path, expires)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         paths = self._list_paths(path_info)
         return any(path_info.path == path for path in paths)
 

dvc/fs/repo.py

@@ -35,9 +35,9 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     PARAM_CHECKSUM = "md5"
 
     def __init__(
-        self, repo, subrepos=False, repo_factory: RepoFactory = None,
+        self, repo=None, subrepos=False, repo_factory: RepoFactory = None
     ):
-        super().__init__(repo, {"url": repo.root_dir})
+        super().__init__()
 
         from dvc.utils.collections import PathStringTrie
 
@@ -49,6 +49,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             self.repo_factory = repo_factory
 
         self._main_repo = repo
+        self.hash_jobs = repo.fs.hash_jobs
         self.root_dir = repo.root_dir
         self._traverse_subrepos = subrepos
 
@@ -61,9 +62,9 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         """Keep a dvcfs instance of each repo."""
 
         if hasattr(repo, "dvc_dir"):
-            self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
+            self._dvcfss[repo.root_dir] = DvcFileSystem(repo=repo)
 
-    def _get_repo(self, path) -> Optional["Repo"]:
+    def _get_repo(self, path: str) -> Optional["Repo"]:
         """Returns repo that the path falls in, using prefix.
 
         If the path is already tracked/collected, it just returns the repo.
@@ -93,11 +94,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             if self._is_dvc_repo(d):
                 repo = self.repo_factory(
                     d,
-                    scm=self.repo.scm,
-                    rev=self.repo.get_rev(),
+                    scm=self._main_repo.scm,
+                    rev=self._main_repo.get_rev(),
                     repo_factory=self.repo_factory,
                 )
-                self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
+                self._dvcfss[repo.root_dir] = DvcFileSystem(repo=repo)
             self._subrepos_trie[d] = repo
 
     def _is_dvc_repo(self, dir_path):
@@ -108,8 +109,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         from dvc.repo import Repo
 
         repo_path = os.path.join(dir_path, Repo.DVC_DIR)
-        # dvcignore will ignore subrepos, therefore using `use_dvcignore=False`
-        return self._main_repo.fs.isdir(repo_path, use_dvcignore=False)
+        return self._main_repo.fs.isdir(repo_path)
 
     def _get_fs_pair(
         self, path
@@ -142,19 +142,20 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs)
 
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        fs, dvc_fs = self._get_fs_pair(path)
+    def exists(self, path_info) -> bool:
+        fs, dvc_fs = self._get_fs_pair(path_info)
 
         if not dvc_fs:
-            return fs.exists(path)
+            return fs.exists(path_info)
 
-        if fs.exists(path):
+        if dvc_fs.repo.dvcignore.is_ignored(fs, path_info):
+            return False
+
+        if fs.exists(path_info):
             return True
 
         try:
-            meta = dvc_fs.metadata(path)
+            meta = dvc_fs.metadata(path_info)
         except FileNotFoundError:
             return False
 
@@ -167,6 +168,9 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def isdir(self, path):  # pylint: disable=arguments-differ
         fs, dvc_fs = self._get_fs_pair(path)
 
+        if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_dir(path):
+            return False
+
         try:
             st = fs.stat(path)
             return stat.S_ISDIR(st.st_mode)
@@ -195,6 +199,9 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def isfile(self, path):  # pylint: disable=arguments-differ
         fs, dvc_fs = self._get_fs_pair(path)
 
+        if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_file(path):
+            return False
+
         try:
             st = fs.stat(path)
             return stat.S_ISREG(st.st_mode)
@@ -238,13 +245,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def _subrepo_walk(self, dir_path, **kwargs):
         """Walk into a new repo.
 
-         NOTE: subrepo will only be discovered when walking if
-         ignore_subrepos is set to False.
+        NOTE: subrepo will only be discovered when walking if
+        ignore_subrepos is set to False.
         """
         fs, dvc_fs = self._get_fs_pair(dir_path)
-        fs_walk = fs.walk(
-            dir_path, topdown=True, ignore_subrepos=not self._traverse_subrepos
-        )
+        fs_walk = fs.walk(dir_path, topdown=True)
         if dvc_fs:
             dvc_walk = dvc_fs.walk(dir_path, topdown=True, **kwargs)
         else:
@@ -306,15 +311,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             elif dirname in repo_set:
                 yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
 
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        dvcfiles=False,
-        follow_subrepos=None,
-        **kwargs
-    ):  # pylint: disable=arguments-differ
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Walk and merge both DVC and repo fss.
 
         Args:
@@ -340,17 +337,14 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 onerror(NotADirectoryError(top))
             return
 
-        ignore_subrepos = not self._traverse_subrepos
-        if follow_subrepos is not None:
-            ignore_subrepos = not follow_subrepos
+        repo = self._get_repo(os.path.abspath(top))
+        dvcfiles = kwargs.pop("dvcfiles", False)
 
         fs, dvc_fs = self._get_fs_pair(top)
         repo_exists = fs.exists(top)
-        repo_walk = fs.walk(
-            top,
-            topdown=topdown,
-            onerror=onerror,
-            ignore_subrepos=ignore_subrepos,
+
+        repo_walk = repo.dvcignore.walk(
+            fs, top, topdown=topdown, onerror=onerror, **kwargs
         )
 
         if not dvc_fs or (repo_exists and dvc_fs.isdvc(top)):
@@ -358,16 +352,20 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return
 
         if not repo_exists:
-            yield from dvc_fs.walk(top, topdown=topdown, **kwargs)
+            yield from dvc_fs.walk(
+                top, topdown=topdown, onerror=onerror, **kwargs
+            )
 
         dvc_walk = None
         if dvc_fs.exists(top):
-            dvc_walk = dvc_fs.walk(top, topdown=topdown, **kwargs)
+            dvc_walk = dvc_fs.walk(
+                top, topdown=topdown, onerror=onerror, **kwargs
+            )
 
         yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
 
-    def walk_files(self, top, **kwargs):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top, **kwargs):
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info, **kwargs):
             for fname in files:
                 yield PathInfo(root) / fname
 
@@ -380,15 +378,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         with open(to_file, "wb+") as to_fobj:
             with Tqdm.wrapattr(
-                to_fobj, "write", desc=name, disable=no_progress_bar,
+                to_fobj, "write", desc=name, disable=no_progress_bar
             ) as wrapped:
                 with self.open(from_info, "rb", **kwargs) as from_fobj:
                     shutil.copyfileobj(from_fobj, wrapped)
 
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        return self._main_repo.fs.hash_jobs
-
     def metadata(self, path):
         abspath = os.path.abspath(path)
         path_info = PathInfo(abspath)

dvc/fs/s3.py

@@ -1,74 +1,40 @@
-import logging
+import functools
 import os
 import threading
-from contextlib import contextmanager
+from collections import defaultdict
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import ConfigError
-from dvc.exceptions import DvcException, ETagMismatchError
-from dvc.hash_info import HashInfo
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
-from dvc.utils import conversions, error_link
 
-from .base import BaseFileSystem
-
-logger = logging.getLogger(__name__)
+from .fsspec_wrapper import ObjectFSWrapper
 
 _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
 
 
-class S3FileSystem(BaseFileSystem):
+# pylint:disable=abstract-method
+class BaseS3FileSystem(ObjectFSWrapper):
     scheme = Schemes.S3
     PATH_CLS = CloudURLInfo
-    REQUIRES = {"boto3": "boto3"}
+    REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    _GRANTS = {
+        "grant_full_control": "GrantFullControl",
+        "grant_read": "GrantRead",
+        "grant_read_acp": "GrantReadACP",
+        "grant_write_acp": "GrantWriteACP",
+    }
+
+    def __init__(self, **config):
+        super().__init__(**config)
 
         url = config.get("url", "s3://")
         self.path_info = self.PATH_CLS(url)
 
-        self.region = config.get("region")
-        self.profile = config.get("profile")
-        self.endpoint_url = config.get("endpointurl")
-
-        self.use_ssl = config.get("use_ssl", True)
-
-        self.extra_args = {}
-
-        self.sse = config.get("sse")
-        if self.sse:
-            self.extra_args["ServerSideEncryption"] = self.sse
-
-        self.sse_kms_key_id = config.get("sse_kms_key_id")
-        if self.sse_kms_key_id:
-            self.extra_args["SSEKMSKeyId"] = self.sse_kms_key_id
-
-        self.acl = config.get("acl")
-        if self.acl:
-            self.extra_args["ACL"] = self.acl
-
-        self._append_aws_grants_to_extra_args(config)
-
-        self.access_key_id = config.get("access_key_id")
-        self.secret_access_key = config.get("secret_access_key")
-        self.session_token = config.get("session_token")
-
-        shared_creds = config.get("credentialpath")
-        if shared_creds:
-            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
-
-        config_path = config.get("configpath")
-        if config_path:
-            os.environ.setdefault("AWS_CONFIG_FILE", config_path)
-        self._transfer_config = None
-
-    # https://github.com/aws/aws-cli/blob/0376c6262d6b15dc36c82e6da6e1aad10249cc8c/awscli/customizations/s3/transferconfig.py#L107-L113
     _TRANSFER_CONFIG_ALIASES = {
         "max_queue_size": "max_io_queue",
         "max_concurrent_requests": "max_concurrency",
@@ -76,11 +42,15 @@ class S3FileSystem(BaseFileSystem):
         "multipart_chunksize": "multipart_chunksize",
     }
 
-    def _transform_config(self, s3_config):
+    def _split_s3_config(self, s3_config):
         """Splits the general s3 config into 2 different config
         objects, one for transfer.TransferConfig and other is the
         general session config"""
 
+        from boto3.s3.transfer import TransferConfig
+
+        from dvc.utils import conversions
+
         config, transfer_config = {}, {}
         for key, value in s3_config.items():
             if key in self._TRANSFER_CONFIG_ALIASES:
@@ -93,370 +63,174 @@ class S3FileSystem(BaseFileSystem):
             else:
                 config[key] = value
 
-        return config, transfer_config
+        # pylint: disable=attribute-defined-outside-init
+        self._transfer_config = TransferConfig(**transfer_config)
+        return config
 
-    def _process_config(self):
-        from boto3.s3.transfer import TransferConfig
+    def _load_aws_config_file(self, profile):
         from botocore.configloader import load_config
 
+        # pylint: disable=attribute-defined-outside-init
+        self._transfer_config = None
         config_path = os.environ.get("AWS_CONFIG_FILE", _AWS_CONFIG_PATH)
         if not os.path.exists(config_path):
             return None
 
         config = load_config(config_path)
-        profile = config["profiles"].get(self.profile or "default")
-        if not profile:
+        profile_config = config["profiles"].get(profile or "default")
+        if not profile_config:
             return None
 
-        s3_config = profile.get("s3", {})
-        s3_config, transfer_config = self._transform_config(s3_config)
-        self._transfer_config = TransferConfig(**transfer_config)
-        return s3_config
+        s3_config = profile_config.get("s3", {})
+        return self._split_s3_config(s3_config)
 
+    def _prepare_credentials(self, **config):
+        from dvc.config import ConfigError
+        from dvc.utils.flatten import flatten, unflatten
+
+        login_info = defaultdict(dict)
+
+        # credentials
+        login_info["key"] = config.get("access_key_id")
+        login_info["secret"] = config.get("secret_access_key")
+        login_info["token"] = config.get("session_token")
+
+        # session configuration
+        login_info["profile"] = config.get("profile")
+        login_info["use_ssl"] = config.get("use_ssl", True)
+
+        # extra client configuration
+        client = login_info["client_kwargs"]
+        client["region_name"] = config.get("region")
+        client["endpoint_url"] = config.get("endpointurl")
+        client["verify"] = config.get("ssl_verify", True)
+
+        # encryptions
+        additional = login_info["s3_additional_kwargs"]
+        additional["ServerSideEncryption"] = config.get("sse")
+        additional["SSEKMSKeyId"] = config.get("sse_kms_key_id")
+        additional["ACL"] = config.get("acl")
+        for grant_option, grant_key in self._GRANTS.items():
+            if config.get(grant_option):
+                if additional["ACL"]:
+                    raise ConfigError(
+                        "`acl` and `grant_*` AWS S3 config options "
+                        "are mutually exclusive"
+                    )
+                additional[grant_key] = config[grant_option]
+
+        # config kwargs
+        session_config = login_info["config_kwargs"]
+        session_config["s3"] = self._load_aws_config_file(
+            login_info["profile"]
+        )
+
+        shared_creds = config.get("credentialpath")
+        if shared_creds:
+            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
+
+        config_path = config.get("configpath")
+        if config_path:
+            os.environ.setdefault("AWS_CONFIG_FILE", config_path)
+
+        return unflatten(
+            {
+                key: value
+                for key, value in flatten(login_info).items()
+                if value is not None
+            }
+        )
+
+    def _entry_hook(self, entry):
+        entry = entry.copy()
+        if "ETag" in entry:
+            entry["etag"] = entry["ETag"].strip('"')
+        return entry
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def fs(self):
+        from s3fs import S3FileSystem as _S3FileSystem
+
+        return _S3FileSystem(**self.fs_args)
+
+
+def _translate_exceptions(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            func(*args, **kwargs)
+        except Exception as exc:
+            from s3fs.errors import translate_boto_error
+
+            raise translate_boto_error(exc)
+
+    return wrapper
+
+
+class S3FileSystem(BaseS3FileSystem):  # pylint:disable=abstract-method
     @wrap_prop(threading.Lock())
     @cached_property
     def s3(self):
         import boto3
 
+        login_info = self.fs_args
+        client_kwargs = login_info.get("client_kwargs", {})
         session_opts = {
-            "profile_name": self.profile,
-            "region_name": self.region,
+            "profile_name": login_info.get("profile"),
+            "region_name": client_kwargs.get("region_name"),
         }
 
-        if self.access_key_id:
-            session_opts["aws_access_key_id"] = self.access_key_id
-        if self.secret_access_key:
-            session_opts["aws_secret_access_key"] = self.secret_access_key
-        if self.session_token:
-            session_opts["aws_session_token"] = self.session_token
+        if "key" in login_info:
+            session_opts["aws_access_key_id"] = login_info["key"]
+        if "secret" in login_info:
+            session_opts["aws_secret_access_key"] = login_info["secret"]
+        if "token" in login_info:
+            session_opts["aws_session_token"] = login_info["token"]
 
         session = boto3.session.Session(**session_opts)
-        s3_config = self._process_config()
 
         return session.resource(
             "s3",
-            endpoint_url=self.endpoint_url,
-            use_ssl=self.use_ssl,
-            config=boto3.session.Config(
-                signature_version="s3v4", s3=s3_config
-            ),
+            endpoint_url=client_kwargs.get("endpoint_url"),
+            use_ssl=login_info["use_ssl"],
         )
 
-    @contextmanager
-    def _get_s3(self):
-        from botocore.exceptions import (
-            EndpointConnectionError,
-            NoCredentialsError,
-        )
-
-        try:
-            yield self.s3
-        except NoCredentialsError as exc:
-            link = error_link("no-credentials")
-            raise DvcException(
-                f"Unable to find AWS credentials. {link}"
-            ) from exc
-        except EndpointConnectionError as exc:
-            link = error_link("connection-error")
-            name = self.endpoint_url or "AWS S3"
-            raise DvcException(
-                f"Unable to connect to '{name}'. {link}"
-            ) from exc
-
-    @contextmanager
-    def _get_bucket(self, bucket):
-        with self._get_s3() as s3:
-            try:
-                yield s3.Bucket(bucket)
-            except s3.meta.client.exceptions.NoSuchBucket as exc:
-                link = error_link("no-bucket")
-                raise DvcException(
-                    f"Bucket '{bucket}' does not exist. {link}"
-                ) from exc
-
-    @contextmanager
     def _get_obj(self, path_info):
-        with self._get_bucket(path_info.bucket) as bucket:
-            try:
-                yield bucket.Object(path_info.path)
-            except bucket.meta.client.exceptions.NoSuchKey as exc:
-                raise DvcException(f"{path_info.url} does not exist") from exc
-
-    def _append_aws_grants_to_extra_args(self, config):
-        # Keys for extra_args can be one of the following list:
-        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
-        """
-          ALLOWED_UPLOAD_ARGS = [
-            'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
-            'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
-            'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
-            'RequestPayer', 'ServerSideEncryption', 'StorageClass',
-            'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
-            'SSEKMSKeyId', 'WebsiteRedirectLocation'
-          ]
-        """
-
-        grants = {
-            "grant_full_control": "GrantFullControl",
-            "grant_read": "GrantRead",
-            "grant_read_acp": "GrantReadACP",
-            "grant_write_acp": "GrantWriteACP",
-        }
-
-        for grant_option, extra_args_key in grants.items():
-            if config.get(grant_option):
-                if self.acl:
-                    raise ConfigError(
-                        "`acl` and `grant_*` AWS S3 config options "
-                        "are mutually exclusive"
-                    )
-
-                self.extra_args[extra_args_key] = config.get(grant_option)
-
-    def _generate_download_url(self, path_info, expires=3600):
-        params = {"Bucket": path_info.bucket, "Key": path_info.path}
-        with self._get_s3() as s3:
-            return s3.meta.client.generate_presigned_url(
-                ClientMethod="get_object",
-                Params=params,
-                ExpiresIn=int(expires),
-            )
-
-    def exists(self, path_info, use_dvcignore=True):
-        """Check if the blob exists. If it does not exist,
-        it could be a part of a directory path.
-
-        eg: if `data/file.txt` exists, check for `data` should return True
-        """
-        return self.isfile(path_info) or self.isdir(path_info)
-
-    def isdir(self, path_info):
-        # S3 doesn't have a concept for directories.
-        #
-        # Using `head_object` with a path pointing to a directory
-        # will throw a 404 error.
-        #
-        # A reliable way to know if a given path is a directory is by
-        # checking if there are more files sharing the same prefix
-        # with a `list_objects` call.
-        #
-        # We need to make sure that the path ends with a forward slash,
-        # since we can end with false-positives like the following example:
-        #
-        #       bucket
-        #       └── data
-        #          ├── alice
-        #          └── alpha
-        #
-        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
-        # While `data/al/` will return nothing.
-        #
-        dir_path = path_info / ""
-        return bool(list(self._list_paths(dir_path, max_items=1)))
-
-    def isfile(self, path_info):
-        if path_info.path.endswith("/"):
-            return False
-
-        return path_info.path in self._list_paths(path_info)
-
-    def info(self, path_info):
-        # temporary workaround, will be removed when migrating to s3fs
-        if self.isdir(path_info):
-            return {"type": "directory"}
-
-        with self._get_obj(path_info) as obj:
-            return {
-                "type": "file",
-                "etag": obj.e_tag.strip('"'),
-                "size": obj.content_length,
-            }
-
-    def _list_paths(self, path_info, max_items=None):
-        with self._get_bucket(path_info.bucket) as bucket:
-            obj_summaries = bucket.objects.filter(Prefix=path_info.path)
-            if max_items is not None:
-                obj_summaries = obj_summaries.page_size(max_items).limit(
-                    max_items
-                )
-            for obj_summary in obj_summaries:
-                yield obj_summary.key
-
-    def walk_files(self, path_info, **kwargs):
-        if not kwargs.pop("prefix", False):
-            path_info = path_info / ""
-        for fname in self._list_paths(path_info, **kwargs):
-            if fname.endswith("/"):
-                continue
-
-            yield path_info.replace(path=fname)
-
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        assert recursive
-
-        with self._get_bucket(path_info.bucket) as bucket:
-            for obj_summary in bucket.objects.filter(Prefix=path_info.path):
-                if detail:
-                    yield {
-                        "type": "file",
-                        "name": obj_summary.key,
-                        "size": obj_summary.size,
-                        "etag": obj_summary.e_tag.strip('"'),
-                    }
-                else:
-                    yield obj_summary.key
-
-    def remove(self, path_info):
-        if path_info.scheme != "s3":
-            raise NotImplementedError
-
-        logger.debug(f"Removing {path_info}")
-        with self._get_obj(path_info) as obj:
-            obj.delete()
-
-    def makedirs(self, path_info):
-        # We need to support creating empty directories, which means
-        # creating an object with an empty body and a trailing slash `/`.
-        #
-        # We are not creating directory objects for every parent prefix,
-        # as it is not required.
-        if not path_info.path:
-            return
-
-        dir_path = path_info / ""
-        with self._get_obj(dir_path) as obj:
-            obj.put(Body="")
-
-    def copy(self, from_info, to_info):
-        with self._get_s3() as s3:
-            self._copy(s3.meta.client, from_info, to_info, self.extra_args)
-
-    @classmethod
-    def _copy_multipart(
-        cls, s3, from_info, to_info, size, n_parts, extra_args
-    ):
-        mpu = s3.create_multipart_upload(
-            Bucket=to_info.bucket, Key=to_info.path, **extra_args
-        )
-        mpu_id = mpu["UploadId"]
-
-        parts = []
-        byte_position = 0
-        for i in range(1, n_parts + 1):
-            obj = s3.head_object(
-                Bucket=from_info.bucket, Key=from_info.path, PartNumber=i
-            )
-            part_size = obj["ContentLength"]
-            lastbyte = byte_position + part_size - 1
-            if lastbyte > size:
-                lastbyte = size - 1
-
-            srange = f"bytes={byte_position}-{lastbyte}"
-
-            part = s3.upload_part_copy(
-                Bucket=to_info.bucket,
-                Key=to_info.path,
-                PartNumber=i,
-                UploadId=mpu_id,
-                CopySourceRange=srange,
-                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
-            )
-            parts.append(
-                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
-            )
-            byte_position += part_size
-
-        assert n_parts == len(parts)
-
-        s3.complete_multipart_upload(
-            Bucket=to_info.bucket,
-            Key=to_info.path,
-            UploadId=mpu_id,
-            MultipartUpload={"Parts": parts},
-        )
-
-    @classmethod
-    def _copy(cls, s3, from_info, to_info, extra_args):
-        # NOTE: object's etag depends on the way it was uploaded to s3 or the
-        # way it was copied within the s3. More specifically, it depends on
-        # the chunk size that was used to transfer it, which would affect
-        # whether an object would be uploaded as a single part or as a
-        # multipart.
-        #
-        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
-        # then it was transferred as a single part, which means that the chunk
-        # size used to transfer it was greater or equal to the ContentLength
-        # of that object. So to preserve that tag over the next transfer, we
-        # could use any value >= ContentLength.
-        #
-        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
-        # then it was transferred as a multipart, which means that the chunk
-        # size used to transfer it was less than ContentLength of that object.
-        # Unfortunately, in general, it doesn't mean that the chunk size was
-        # the same throughout the transfer, so it means that in order to
-        # preserve etag, we need to transfer each part separately, so the
-        # object is transfered in the same chunks as it was originally.
-        from boto3.s3.transfer import TransferConfig
-
-        obj = s3.head_object(Bucket=from_info.bucket, Key=from_info.path)
-        etag = obj["ETag"].strip('"')
-        size = obj["ContentLength"]
-
-        _, _, parts_suffix = etag.partition("-")
-        if parts_suffix:
-            n_parts = int(parts_suffix)
-            cls._copy_multipart(
-                s3, from_info, to_info, size, n_parts, extra_args=extra_args
-            )
-        else:
-            source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(
-                source,
-                to_info.bucket,
-                to_info.path,
-                ExtraArgs=extra_args,
-                Config=TransferConfig(multipart_threshold=size + 1),
-            )
-
-        cached_etag = s3.head_object(Bucket=to_info.bucket, Key=to_info.path)[
-            "ETag"
-        ].strip('"')
-        if etag != cached_etag:
-            raise ETagMismatchError(etag, cached_etag)
-
-    def etag(self, path_info):
-        with self._get_obj(path_info) as obj:
-            return HashInfo("etag", size=obj.content_length,)
-
-    def _upload_fobj(self, fobj, to_info):
-        with self._get_obj(to_info) as obj:
-            obj.upload_fileobj(fobj, Config=self._transfer_config)
+        bucket = self.s3.Bucket(path_info.bucket)
+        return bucket.Object(path_info.path)
 
+    @_translate_exceptions
     def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+        self, from_file, to_info, name=None, no_progress_bar=False, **pbar_args
     ):
-        with self._get_obj(to_info) as obj:
-            total = os.path.getsize(from_file)
-            with Tqdm(
-                disable=no_progress_bar, total=total, bytes=True, desc=name
-            ) as pbar:
-                obj.upload_file(
-                    from_file,
-                    Callback=pbar.update,
-                    ExtraArgs=self.extra_args,
-                    Config=self._transfer_config,
-                )
+        total = os.path.getsize(from_file)
+        with Tqdm(
+            disable=no_progress_bar,
+            total=total,
+            bytes=True,
+            desc=name,
+            **pbar_args,
+        ) as pbar:
+            obj = self._get_obj(to_info)
+            obj.upload_file(
+                from_file,
+                Callback=pbar.update,
+                ExtraArgs=self.fs_args.get("s3_additional_kwargs"),
+                Config=self._transfer_config,
+            )
+        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
 
-    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        with self._get_obj(from_info) as obj:
-            with Tqdm(
-                disable=no_progress_bar,
-                total=obj.content_length,
-                bytes=True,
-                desc=name,
-            ) as pbar:
-                obj.download_file(
-                    to_file, Callback=pbar.update, Config=self._transfer_config
-                )
+    @_translate_exceptions
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
+    ):
+        obj = self._get_obj(from_info)
+        with Tqdm(
+            disable=no_progress_bar,
+            total=obj.content_length,
+            bytes=True,
+            desc=name,
+            **pbar_args,
+        ) as pbar:
+            obj.download_file(to_file, Callback=pbar.update)

dvc/fs/ssh/__init__.py

@@ -9,7 +9,7 @@ from urllib.parse import urlparse
 
 from funcy import first, memoize, silent, wrap_with
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.hash_info import HashInfo
 from dvc.scheme import Schemes
 
@@ -30,10 +30,10 @@ def ask_password(host, user, port):
     )
 
 
-class SSHFileSystem(BaseFileSystem):
+class SSHFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
-    JOBS = 4
+    _JOBS = 4
 
     PARAM_CHECKSUM = "md5"
     # At any given time some of the connections will go over network and
@@ -46,8 +46,8 @@ class SSHFileSystem(BaseFileSystem):
     DEFAULT_PORT = 22
     TIMEOUT = 1800
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
         url = config.get("url")
         if url:
             parsed = urlparse(url)
@@ -159,7 +159,7 @@ class SSHFileSystem(BaseFileSystem):
             else:
                 yield io.TextIOWrapper(fd, encoding=encoding)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         with self.ssh(path_info) as ssh:
             return ssh.exists(path_info.path)
 

dvc/fs/ssh/connection.py

@@ -17,14 +17,14 @@ from ..base import RemoteCmdError
 try:
     import paramiko
 except ImportError:
-    paramiko = None
+    paramiko = None  # type: ignore
 
 
 logger = logging.getLogger(__name__)
 
 
 def sizeof_fmt(num, suffix="B"):
-    """ Convert number of bytes to human-readable string """
+    """Convert number of bytes to human-readable string"""
     for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
         if abs(num) < 1024.0:
             return f"{num:3.1f}{unit}{suffix}"

dvc/fs/webdav.py

@@ -46,9 +46,9 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     DETAIL_FIELDS = frozenset(("etag", "size"))
 
     # Constructor
-    def __init__(self, repo, config):
+    def __init__(self, **config):
         # Call BaseFileSystem constructor
-        super().__init__(repo, config)
+        super().__init__(**config)
 
         # Get username from configuration
         self.user = config.get("user", None)
@@ -117,7 +117,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
         # Check whether connection is valid (root should always exist)
-        if not client.check("/"):
+        if not client.check(self.path_info.path):
             raise WebDAVConnectionError(self.hostname)
 
         return client
@@ -142,7 +142,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         return io.TextIOWrapper(fobj, encoding=encoding)
 
     # Checks whether file/directory exists at remote
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         # Use webdav check to test for file existence
         return self._client.check(path_info.path)
 
@@ -151,33 +151,30 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # Use webdav is_dir to test whether path points to a directory
         return self._client.is_dir(path_info.path)
 
-    # Yields path info to all files
     def walk_files(self, path_info, **kwargs):
-        # Check whether directory exists
         if not self.exists(path_info):
             return
 
-        # Collect directories
-        dirs = deque([path_info.path])
+        for file in self.find(path_info):
+            yield path_info.replace(path=file)
 
-        # Iterate all directories found so far
-        while dirs:
-            # Iterate directory content
-            for entry in self._client.list(dirs.pop(), get_info=True):
-                # Construct path_info to entry
-                info = path_info.replace(path=entry["path"])
-
-                # Check whether entry is a directory
+    def ls(self, path_info, detail=False):
+        for entry in self._client.list(path_info.path):
+            path = entry["path"]
+            if detail:
                 if entry["isdir"]:
-                    # Append new found directory to directory list
-                    dirs.append(info.path)
+                    yield {"type": "directory", "name": path}
                 else:
-                    # Yield path info to non directory
-                    yield info
+                    yield {
+                        "type": "file",
+                        "name": path,
+                        "size": entry["size"],
+                        "etag": entry["etag"],
+                    }
+            else:
+                yield path
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
+    def find(self, path_info, detail=False):
         dirs = deque([path_info.path])
 
         while dirs:
@@ -197,14 +194,6 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 else:
                     yield path
 
-            if not recursive:
-                for entry in dirs:
-                    if detail:
-                        yield {"type": "directory", "name": entry}
-                    else:
-                        yield entry
-                return None
-
     # Removes file/directory
     def remove(self, path_info):
         # Use webdav client clean (DELETE) method to remove file/directory
@@ -213,7 +202,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     # Creates directories
     def makedirs(self, path_info):
         # Terminate recursion
-        if path_info.path == "/" or self.exists(path_info):
+        if path_info.path == self.path_info.path or self.exists(path_info):
             return
 
         # Recursively descent to root
@@ -252,7 +241,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
     # Uploads file to remote
     def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs,
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
         # First try to create parent directories
         self.makedirs(to_info.parent)
@@ -278,7 +267,4 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
     def info(self, path_info):
         info = self._client.info(path_info.path)
-        return {
-            "size": int(info["size"]),
-            "etag": info["etag"],
-        }
+        return {"size": int(info["size"]), "etag": info["etag"]}

dvc/fs/webhdfs.py

@@ -30,15 +30,15 @@ def update_pbar(pbar, total):
     return update
 
 
-class WebHDFSFileSystem(BaseFileSystem):
+class WebHDFSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     scheme = Schemes.WEBHDFS
     PATH_CLS = CloudURLInfo
     REQUIRES = {"hdfs": "hdfs"}
     PARAM_CHECKSUM = "checksum"
     TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
         self.path_info = None
         url = config.get("url")
@@ -114,7 +114,7 @@ class WebHDFSFileSystem(BaseFileSystem):
 
         self.hdfs_client.delete(path_info.path)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         assert not isinstance(path_info, list)
         assert path_info.scheme == "webhdfs"
 

dvc/ignore.py

@@ -7,9 +7,12 @@ from itertools import groupby, takewhile
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
 
+from dvc.fs.base import BaseFileSystem
 from dvc.path_info import PathInfo
 from dvc.pathspec_math import PatternInfo, merge_patterns
+from dvc.scheme import Schemes
 from dvc.system import System
+from dvc.types import AnyPath, List, Optional
 from dvc.utils import relpath
 from dvc.utils.collections import PathStringTrie
 
@@ -49,16 +52,12 @@ class DvcIgnorePatterns(DvcIgnore):
         ]
 
     @classmethod
-    def from_files(cls, ignore_file_path, fs):
-        assert os.path.isabs(ignore_file_path)
-        dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-        ignore_file_rel_path = os.path.relpath(ignore_file_path, fs.fs_root)
-        with fs.open(ignore_file_path, encoding="utf-8") as fobj:
+    def from_file(cls, path, fs, name):
+        assert os.path.isabs(path)
+        dirname = os.path.normpath(os.path.dirname(path))
+        with fs.open(path, encoding="utf-8") as fobj:
             path_spec_lines = [
-                PatternInfo(
-                    line,
-                    "{}:{}:{}".format(ignore_file_rel_path, line_no + 1, line),
-                )
+                PatternInfo(line, "{}:{}:{}".format(name, line_no + 1, line))
                 for line_no, line in enumerate(
                     map(str.strip, fobj.readlines())
                 )
@@ -67,7 +66,7 @@ class DvcIgnorePatterns(DvcIgnore):
 
         return cls(path_spec_lines, dirname)
 
-    def __call__(self, root, dirs, files):
+    def __call__(self, root: List[str], dirs: List[str], files: List[str]):
         files = [f for f in files if not self.matches(root, f)]
         dirs = [d for d in dirs if not self.matches(root, d, True)]
 
@@ -159,56 +158,48 @@ def _no_match(path):
     return CheckIgnoreResult(path, False, ["::"])
 
 
-class DvcIgnoreFilterNoop:
-    def __init__(self, fs, root_dir):
-        pass
-
-    def __call__(self, root, dirs, files, **kwargs):
-        return dirs, files
-
-    def is_ignored_dir(self, _):
-        return False
-
-    def is_ignored_file(self, _):
-        return False
-
-    def check_ignore(self, path):
-        return _no_match(path)
-
-    def is_ignored(self, _):
-        return False
-
-
 class DvcIgnoreFilter:
     def __init__(self, fs, root_dir):
         from dvc.repo import Repo
 
-        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+        default_ignore_patterns = [
+            ".hg/",
+            ".git/",
+            ".git",
+            "{}/".format(Repo.DVC_DIR),
+        ]
 
         self.fs = fs
         self.root_dir = root_dir
         self.ignores_trie_fs = PathStringTrie()
+        self._ignores_trie_subrepos = PathStringTrie()
         self.ignores_trie_fs[root_dir] = DvcIgnorePatterns(
             default_ignore_patterns, root_dir
         )
-        self._ignored_subrepos = PathStringTrie()
-        self._update(self.root_dir)
+        self._ignores_trie_subrepos[root_dir] = self.ignores_trie_fs[root_dir]
+        self._update(
+            self.root_dir,
+            self._ignores_trie_subrepos,
+            dnames=None,
+            ignore_subrepos=False,
+        )
+        self._update(
+            self.root_dir,
+            self.ignores_trie_fs,
+            dnames=None,
+            ignore_subrepos=True,
+        )
 
-    def _update(self, dirname):
-        self._update_sub_repo(dirname)
-
-        old_pattern = self.ignores_trie_fs.longest_prefix(dirname).value
+    def _update_trie(self, dirname: str, trie: PathStringTrie) -> None:
+        old_pattern = trie.longest_prefix(dirname).value
         matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
 
-        ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if not matches and self.fs.exists(
-            ignore_file_path, use_dvcignore=False
-        ):
-            new_pattern = DvcIgnorePatterns.from_files(
-                ignore_file_path, self.fs
-            )
+        path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
+        if not matches and self.fs.exists(path):
+            name = os.path.relpath(path, self.root_dir)
+            new_pattern = DvcIgnorePatterns.from_file(path, self.fs, name)
             if old_pattern:
-                self.ignores_trie_fs[dirname] = DvcIgnorePatterns(
+                trie[dirname] = DvcIgnorePatterns(
                     *merge_patterns(
                         old_pattern.pattern_list,
                         old_pattern.dirname,
@@ -217,11 +208,32 @@ class DvcIgnoreFilter:
                     )
                 )
             else:
-                self.ignores_trie_fs[dirname] = new_pattern
+                trie[dirname] = new_pattern
         elif old_pattern:
-            self.ignores_trie_fs[dirname] = old_pattern
+            trie[dirname] = old_pattern
 
-    def _update_sub_repo(self, path):
+    def _update(
+        self,
+        dirname: str,
+        ignore_trie: PathStringTrie,
+        dnames: Optional["List"],
+        ignore_subrepos: bool,
+    ) -> None:
+        self._update_trie(dirname, ignore_trie)
+
+        if ignore_subrepos:
+            if dnames is None:
+                try:
+                    _, dnames, _ = next(self.fs.walk(dirname))
+                except StopIteration:
+                    dnames = []
+
+            for dname in dnames:
+                self._update_sub_repo(
+                    os.path.join(dirname, dname), ignore_trie
+                )
+
+    def _update_sub_repo(self, path, ignore_trie: PathStringTrie):
         from dvc.repo import Repo
 
         if path == self.root_dir:
@@ -232,14 +244,11 @@ class DvcIgnoreFilter:
             return
 
         root, dname = os.path.split(path)
-        self._ignored_subrepos[root] = self._ignored_subrepos.get(
-            root, set()
-        ) | {dname}
         pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
         new_pattern = DvcIgnorePatterns([pattern_info], root)
-        old_pattern = self.ignores_trie_fs.longest_prefix(root).value
+        old_pattern = ignore_trie.longest_prefix(root).value
         if old_pattern:
-            self.ignores_trie_fs[root] = DvcIgnorePatterns(
+            ignore_trie[root] = DvcIgnorePatterns(
                 *merge_patterns(
                     old_pattern.pattern_list,
                     old_pattern.dirname,
@@ -248,25 +257,50 @@ class DvcIgnoreFilter:
                 )
             )
         else:
-            self.ignores_trie_fs[root] = new_pattern
+            ignore_trie[root] = new_pattern
 
     def __call__(self, root, dirs, files, ignore_subrepos=True):
-        for dname in dirs:
-            self._update_sub_repo(os.path.join(root, dname))
-
-        ignore_pattern = self._get_trie_pattern(root)
+        abs_root = os.path.abspath(root)
+        ignore_pattern = self._get_trie_pattern(
+            abs_root, dnames=dirs, ignore_subrepos=ignore_subrepos
+        )
         if ignore_pattern:
-            dirs, files = ignore_pattern(root, dirs, files)
-            if not ignore_subrepos:
-                dirs.extend(self._ignored_subrepos.get(root, []))
+            dirs, files = ignore_pattern(abs_root, dirs, files)
         return dirs, files
 
-    def _get_trie_pattern(self, dirname):
-        ignore_pattern = self.ignores_trie_fs.get(dirname)
+    def walk(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
+        ignore_subrepos = kwargs.pop("ignore_subrepos", True)
+        if fs.scheme == Schemes.LOCAL:
+            for root, dirs, files in fs.walk(path_info, **kwargs):
+                dirs[:], files[:] = self(
+                    root, dirs, files, ignore_subrepos=ignore_subrepos
+                )
+                yield root, dirs, files
+        else:
+            yield from fs.walk(path_info, **kwargs)
+
+    def walk_files(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
+        if fs.scheme == Schemes.LOCAL:
+            for root, _, files in self.walk(fs, path_info, **kwargs):
+                for file in files:
+                    # NOTE: os.path.join is ~5.5 times slower
+                    yield PathInfo(f"{root}{os.sep}{file}")
+        else:
+            yield from fs.walk_files(path_info)
+
+    def _get_trie_pattern(
+        self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True
+    ) -> Optional["DvcIgnorePatterns"]:
+        if ignore_subrepos:
+            ignores_trie = self.ignores_trie_fs
+        else:
+            ignores_trie = self._ignores_trie_subrepos
+
+        ignore_pattern = ignores_trie.get(dirname)
         if ignore_pattern:
             return ignore_pattern
 
-        prefix = self.ignores_trie_fs.longest_prefix(dirname).key
+        prefix = ignores_trie.longest_prefix(dirname).key
         if not prefix:
             # outside of the repo
             return None
@@ -281,33 +315,31 @@ class DvcIgnoreFilter:
         dirs.append(dirname)
 
         for parent in dirs:
-            self._update(parent)
+            self._update(parent, ignores_trie, dnames, ignore_subrepos)
 
-        return self.ignores_trie_fs.get(dirname)
+        return ignores_trie.get(dirname)
 
-    def _is_ignored(self, path, is_dir=False):
+    def _is_ignored(
+        self, path: str, is_dir: bool = False, ignore_subrepos: bool = True
+    ):
         if self._outside_repo(path):
             return False
         dirname, basename = os.path.split(os.path.normpath(path))
-        ignore_pattern = self._get_trie_pattern(dirname)
+        ignore_pattern = self._get_trie_pattern(dirname, None, ignore_subrepos)
         if ignore_pattern:
             return ignore_pattern.matches(dirname, basename, is_dir)
         return False
 
-    def _is_subrepo(self, path):
-        dirname, basename = os.path.split(os.path.normpath(path))
-        return basename in self._ignored_subrepos.get(dirname, set())
-
-    def is_ignored_dir(self, path, ignore_subrepos=True):
+    def is_ignored_dir(self, path: str, ignore_subrepos: bool = True) -> bool:
+        "Only used in LocalFileSystem"
         path = os.path.abspath(path)
-        if not ignore_subrepos:
-            return not self._is_subrepo(path)
         if path == self.root_dir:
             return False
 
-        return self._is_ignored(path, True)
+        return self._is_ignored(path, True, ignore_subrepos=ignore_subrepos)
 
-    def is_ignored_file(self, path):
+    def is_ignored_file(self, path: str) -> bool:
+        "Only used in LocalFileSystem"
         path = os.path.abspath(path)
         return self._is_ignored(path, False)
 
@@ -334,21 +366,27 @@ class DvcIgnoreFilter:
             pattern = self._get_trie_pattern(dirname)
             if pattern:
                 matches = pattern.matches(
-                    dirname, basename, os.path.isdir(full_target), True,
+                    dirname, basename, os.path.isdir(full_target), True
                 )
 
                 if matches:
                     return CheckIgnoreResult(target, True, matches)
         return _no_match(target)
 
-    def is_ignored(self, path):
+    def is_ignored(
+        self, fs: BaseFileSystem, path: str, ignore_subrepos: bool = True
+    ) -> bool:
         # NOTE: can't use self.check_ignore(path).match for now, see
         # https://github.com/iterative/dvc/issues/4555
-        if os.path.isfile(path):
+        if fs.scheme != Schemes.LOCAL:
+            return False
+        if fs.isfile(path):
             return self.is_ignored_file(path)
-        if os.path.isdir(path):
-            return self.is_ignored_dir(path)
-        return self.is_ignored_file(path) or self.is_ignored_dir(path)
+        if fs.isdir(path):
+            return self.is_ignored_dir(path, ignore_subrepos)
+        return self.is_ignored_file(path) or self.is_ignored_dir(
+            path, ignore_subrepos
+        )
 
 
 def init(path):

dvc/info.py

@@ -31,34 +31,32 @@ def get_dvc_info():
     ]
 
     try:
-        repo = Repo()
+        with Repo() as repo:
+            # cache_dir might not exist yet (e.g. after `dvc init`), and we
+            # can't auto-create it, as it might cause issues if the user
+            # later decides to enable shared cache mode with
+            # `dvc config cache.shared group`.
+            if os.path.exists(repo.odb.local.cache_dir):
+                info.append(
+                    "Cache types: {}".format(_get_linktype_support_info(repo))
+                )
+                fs_type = get_fs_type(repo.odb.local.cache_dir)
+                info.append(f"Cache directory: {fs_type}")
+            else:
+                info.append("Cache types: " + error_link("no-dvc-cache"))
 
-        # cache_dir might not exist yet (e.g. after `dvc init`), and we
-        # can't auto-create it, as it might cause issues if the user
-        # later decides to enable shared cache mode with
-        # `dvc config cache.shared group`.
-        if os.path.exists(repo.odb.local.cache_dir):
-            info.append(
-                "Cache types: {}".format(_get_linktype_support_info(repo))
-            )
-            fs_type = get_fs_type(repo.odb.local.cache_dir)
-            info.append(f"Cache directory: {fs_type}")
-        else:
-            info.append("Cache types: " + error_link("no-dvc-cache"))
-
-        info.append(f"Caches: {_get_caches(repo.odb)}")
-
-        info.append(f"Remotes: {_get_remotes(repo.config)}")
+            info.append(f"Caches: {_get_caches(repo.odb)}")
+            info.append(f"Remotes: {_get_remotes(repo.config)}")
 
+            root_directory = repo.root_dir
+            fs_root = get_fs_type(os.path.abspath(root_directory))
+            info.append(f"Workspace directory: {fs_root}")
+            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
     except NotDvcRepoError:
         pass
     except SCMError:
         info.append("Repo: dvc, git (broken)")
-    else:
-        root_directory = repo.root_dir
-        fs_root = get_fs_type(os.path.abspath(root_directory))
-        info.append(f"Workspace directory: {fs_root}")
-        info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
+
     return "\n".join(info)
 
 

dvc/istextfile.py

@@ -23,10 +23,10 @@ def istextblock(block):
 
 
 def istextfile(fname, fs, blocksize=512):
-    """ Uses heuristics to guess whether the given file is text or binary,
-        by reading a single block of bytes from the file.
-        If more than 30% of the chars in the block are non-text, or there
-        are NUL ('\x00') bytes in the block, assume this is a binary file.
+    """Uses heuristics to guess whether the given file is text or binary,
+    by reading a single block of bytes from the file.
+    If more than 30% of the chars in the block are non-text, or there
+    are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
     with fs.open(fname, "rb") as fobj:
         block = fobj.read(blocksize)

dvc/logger.py

@@ -188,8 +188,8 @@ def _stack_trace(exc_info):
 
 def disable_other_loggers():
     logging.captureWarnings(True)
-    root = logging.root
-    for (logger_name, logger) in root.manager.loggerDict.items():
+    loggerDict = logging.root.manager.loggerDict  # pylint: disable=no-member
+    for logger_name, logger in loggerDict.items():
         if logger_name != "dvc" and not logger_name.startswith("dvc."):
             logger.disabled = True
 
@@ -197,6 +197,17 @@ def disable_other_loggers():
 def setup(level=logging.INFO):
     colorama.init()
 
+    if level >= logging.DEBUG:
+        # Unclosed session errors for asyncio/aiohttp are only available
+        # on the tracing mode for extensive debug purposes. They are really
+        # noisy, and this is potentially somewhere in the client library
+        # not managing their own session. Even though it is the best practice
+        # for them to do so, we can be assured that these errors raised when
+        # the object is getting deallocated, so no need to take any extensive
+        # action.
+        logging.getLogger("asyncio").setLevel(logging.CRITICAL)
+        logging.getLogger("aiohttp").setLevel(logging.CRITICAL)
+
     addLoggingLevel("TRACE", logging.DEBUG - 5)
     logging.config.dictConfig(
         {
@@ -245,7 +256,7 @@ def setup(level=logging.INFO):
                         "console_trace",
                         "console_errors",
                     ],
-                },
+                }
             },
             "disable_existing_loggers": False,
         }

dvc/main.py

@@ -52,7 +52,7 @@ def main(argv=None):  # noqa: C901
 
         with debugtools(args):
             cmd = args.func(args)
-            ret = cmd.run()
+            ret = cmd.do_run()
     except ConfigError:
         logger.exception("configuration error")
         ret = 251

dvc/objects/__init__.py

@@ -1,4 +1,5 @@
 import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from dvc.progress import Tqdm
 
@@ -7,10 +8,26 @@ from .tree import Tree
 logger = logging.getLogger(__name__)
 
 
-def save(odb, obj, **kwargs):
+def save(odb, obj, jobs=None, **kwargs):
     if isinstance(obj, Tree):
-        for _, entry in Tqdm(obj):
-            odb.add(entry.path_info, entry.fs, entry.hash_info, **kwargs)
+        with ThreadPoolExecutor(max_workers=jobs) as executor:
+            for future in Tqdm(
+                as_completed(
+                    executor.submit(
+                        odb.add,
+                        entry.path_info,
+                        entry.fs,
+                        entry.hash_info,
+                        **kwargs
+                    )
+                    for _, entry in obj
+                ),
+                desc="Saving files",
+                total=len(obj),
+                unit="file",
+            ):
+                future.result()
+
     odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)