dvc: bump to 2.7.1

* Set timeout for jobs in Github Actions

Sometimes tests are getting frozen(most likely an issue with `pytest-xdist`), and sometimes the pip resolver is slow due to some weird constraints set by us or one of our dependencies.

The default timeout is 360 minutes (6hrs), but our test completes on ~18 minutes at maximum, so I think 30mins should be enough to account for occasional CI slowdown.

Also keeping the total job timeout at 45 mins, we usually require just 4 mins of extra time to do installation and setup but giving it 11 more minutes just in case.

Similarly, `lint` completes in <5 mins usually, but giving it a 10 mins timeout.

This should give us early feedback on the issue.

* Update setup.cfg

* Update setup.cfg

.codecov.yml

@@ -3,7 +3,7 @@ coverage:
   status:
     project:
       default:
-        target: 85
+        target: 90
         threshold: 10%
     patch:
       default:

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @iterative/dvc

.github/workflows/packages.yaml

@@ -5,9 +5,10 @@ on:
     - master
   release:
     types: [released, prereleased]
+  workflow_dispatch:
 jobs:
   deb:
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-18.04
     steps:
     - uses: actions/[email protected]
     - name: Set up Python 3.8
@@ -15,7 +16,7 @@ jobs:
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
+      uses: actions/setup-ruby@v1.1.3
       with:
         ruby-version: 2.4
     - name: Install deps
@@ -36,7 +37,7 @@ jobs:
         asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_content_type: binary/octet-stream
   rpm:
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-18.04
     steps:
     - uses: actions/[email protected]
     - name: Set up Python 3.8
@@ -44,7 +45,7 @@ jobs:
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
+      uses: actions/setup-ruby@v1.1.3
       with:
         ruby-version: 2.4
     - name: Install deps
@@ -73,7 +74,7 @@ jobs:
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
+      uses: actions/setup-ruby@v1.1.3
       with:
         ruby-version: 2.4
     - name: Install deps
@@ -125,7 +126,7 @@ jobs:
         asset_name: dvc-${{ github.event.release.tag_name }}.exe
         asset_content_type: binary/octet-stream
   pip:
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-18.04
     steps:
     - uses: actions/[email protected]
     - name: Set up Python 3.7

.github/workflows/tests.yaml

@@ -1,12 +1,10 @@
 name: Tests
 
 on:
-  pull_request:
-    branches:
-      - master
-  push:
-    branches:
-      - master
+  pull_request: {}
+  push: {}
+  schedule:
+    - cron: '5 1 * * *'  # every day at 01:05
 
 env:
   DVC_TEST: "true"
@@ -15,6 +13,7 @@ env:
 
 jobs:
   lint:
+    timeout-minutes: 10
     runs-on: ubuntu-latest
     steps:
     - uses: actions/[email protected]
@@ -25,7 +24,7 @@ jobs:
     - name: Install requirements
       run: |
         pip install wheel
-        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+        pip install ".[all,tests,terraform]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
@@ -34,19 +33,21 @@ jobs:
       run: python setup.py checkdocs
     - uses: pre-commit/[email protected]
   tests:
+    timeout-minutes: 45
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-18.04, windows-2019, macos-10.15]
         pyv: ["3.6", "3.7", "3.8", "3.9"]
-# macos 11.0 workers take too long to start up
-#        include:
-#          - os: "macos-11.0"
-#            pyv: "3.9"
+        include:
+          - os: "macos-10.15"
+            pyv: "3.10.0-rc.1"
         exclude:
           - os: macos-10.15
             pyv: "3.9"
+          - os: macos-10.15
+            pyv: "3.7"
 
     steps:
     - uses: actions/[email protected]
@@ -60,17 +61,21 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/[email protected].5
+      uses: actions/[email protected].6
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
         restore-keys: |
           ${{ runner.os }}-pip-
+    - name: install libgit2 on Python3.10 in macOS
+      if: matrix.pyv == '3.10.0-rc.1' && matrix.os == 'macos-10.15'
+      run: |
+        brew install libgit2
     - name: install
       if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
-        pip install wheel
-        pip install -e ".[all,tests]"
+        pip install --upgrade pip setuptools wheel
+        pip install -e ".[all,tests,terraform]"
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
@@ -87,14 +92,42 @@ jobs:
       run: |
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
+    - name: disable gdrive tests
+      if: >-
+        github.event_name != 'schedule' &&
+        (matrix.pyv != '3.8' || matrix.os != 'ubuntu-18.04')
+      run: |
+        echo "extra_test_args=--disable-gdrive" >> $GITHUB_ENV
     - name: run tests
+      timeout-minutes: 30
       env:
         GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      run: python -m tests --all --cov-report=xml --cov-report=term -n=4
+        OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID}}
+        OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET}}
+        OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT}}
+      run: >-
+        python -m tests -n=4
+        --cov-report=xml --cov-report=term
+        --all ${{ env.extra_test_args }}
     - name: upload coverage report
-      uses: codecov/codecov-action@v1.4.1
+      uses: codecov/codecov-action@v2.0.3
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
+  notify:
+    if: github.ref == 'refs/heads/master' && failure()
+    needs:
+      - lint
+      - tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Slack Notification
+      uses: rtCamp/[email protected]
+      env:
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        SLACK_COLOR: ${{ job.status }}
+        SLACK_MESSAGE: 'CI Failed on master :boom:'
+        SLACK_TITLE: CI Status
+        SLACK_USERNAME: DVC-CI

.pre-commit-config.yaml

@@ -1,14 +1,24 @@
+ci:
+  skip: [pylint, dvc-pre-commit]
+
 repos:
   - hooks:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 19.10b0
+    rev: 21.8b0
+  - hooks:
+      - id: codespell
+        args: 
+          - --ignore-words-list
+          - ba,datas,fo,uptodate
+    repo: https://github.com/codespell-project/codespell
+    rev: v2.1.0
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.8.0
+    rev: 5.9.3
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,13 +28,18 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.0
+    rev: 3.9.2
   - hooks:
       -  id: mypy
          language_version: python3
          files: ^dvc/
+         additional_dependencies:
+           - types-requests
+           - types-paramiko
+           - types-tabulate
+           - types-toml
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.812'
+    rev: 'v0.910'
   - repo: local
     hooks:
     - id: pylint
@@ -39,7 +54,7 @@ repos:
         id: beautysh
         language_version: python3
     repo: https://github.com/lovesegfault/beautysh
-    rev: 6.0.1
+    rev: v6.1.0
   - hooks:
       - args:
           - git-hook

.pylintrc

@@ -313,6 +313,7 @@ missing-member-hint-distance=1
 # showing a hint for a missing member.
 missing-member-max-choices=1
 
+signature-mutators=funcy.decorators.decorator,
 
 [VARIABLES]
 

README.rst

@@ -8,7 +8,7 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
+|CI| |Maintainability| |Coverage| |Donate| |DOI|
 
 |PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
 
@@ -20,8 +20,9 @@ learning projects. Key features:
 #. Simple **command line** Git-like experience. Does not require installing and maintaining
    any databases. Does not depend on any proprietary online services.
 
-#. Management and versioning of **datasets** and **machine learning models**. Data is saved in
-   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
+#. Management and versioning of **datasets** and **machine learning
+   models**. Data can be saved in S3, Google cloud, Azure, Alibaba 
+   cloud, SSH server, HDFS, or even local HDD RAID.
 
 #. Makes projects **reproducible** and **shareable**; helping to answer questions about how
    a model was built.
@@ -29,9 +30,9 @@ learning projects. Key features:
 #. Helps manage experiments with Git tags/branches and **metrics** tracking.
 
 **DVC** aims to replace spreadsheet and document sharing tools (such as Excel or Google Docs)
-which are being used frequently as both knowledge repositories and team ledgers.
-DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions;
-as well as ad-hoc data file suffixes and prefixes.
+frequently used as both knowledge repositories and team ledgers.
+DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions
+and ad-hoc data file suffixes and prefixes.
 
 .. contents:: **Contents**
   :backlinks: none
@@ -65,25 +66,25 @@ Quick start
 
 Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include:
 
-+-----------------------------------+-------------------------------------------------------------------+
-| Step                              | Command                                                           |
-+===================================+===================================================================+
-| Track data                        | | ``$ git add train.py``                                          |
-|                                   | | ``$ dvc add images.zip``                                        |
-+-----------------------------------+-------------------------------------------------------------------+
-| Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
-|                                   | | ``$ dvc run -d images/ -d train.py -o model.p python train.py`` |
-+-----------------------------------+-------------------------------------------------------------------+
-| Make changes and reproduce        | | ``$ vi train.py``                                               |
-|                                   | | ``$ dvc repro model.p.dvc``                                     |
-+-----------------------------------+-------------------------------------------------------------------+
-| Share code                        | | ``$ git add .``                                                 |
-|                                   | | ``$ git commit -m 'The baseline model'``                        |
-|                                   | | ``$ git push``                                                  |
-+-----------------------------------+-------------------------------------------------------------------+
-| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``        |
-|                                   | | ``$ dvc push``                                                  |
-+-----------------------------------+-------------------------------------------------------------------+
++-----------------------------------+----------------------------------------------------------------------------+
+| Step                              | Command                                                                    |
++===================================+============================================================================+
+| Track data                        | | ``$ git add train.py``                                                   |
+|                                   | | ``$ dvc add images.zip``                                                 |
++-----------------------------------+----------------------------------------------------------------------------+
+| Connect code and data by commands | | ``$ dvc run -n prepare -d images.zip -o images/ unzip -q images.zip``    |
+|                                   | | ``$ dvc run -n train -d images/ -d train.py -o model.p python train.py`` |
++-----------------------------------+----------------------------------------------------------------------------+
+| Make changes and reproduce        | | ``$ vi train.py``                                                        |
+|                                   | | ``$ dvc repro model.p.dvc``                                              |
++-----------------------------------+----------------------------------------------------------------------------+
+| Share code                        | | ``$ git add .``                                                          |
+|                                   | | ``$ git commit -m 'The baseline model'``                                 |
+|                                   | | ``$ git push``                                                           |
++-----------------------------------+----------------------------------------------------------------------------+
+| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``                 |
+|                                   | | ``$ dvc push``                                                           |
++-----------------------------------+----------------------------------------------------------------------------+
 
 Installation
 ============
@@ -182,20 +183,38 @@ Fedora / CentOS (rpm)
 Comparison to related technologies
 ==================================
 
+#. Data Engineering tools such as `AirFlow <https://airflow.apache.org/>`,
+   `Luigi <https://github.com/spotify/luigi>`, and others - in DVC data,
+   model and ML pipelines represent a single ML project focused on data
+   scientists' experience.  Data engineering tools orchestrate multiple data
+   projects and focus on efficient execution. A DVC project can be used from
+   existing data pipelines as a single execution step.
+
 #. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
    not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
    copying/duplicating files.
 
-#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
-   etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files
-   much more efficiently.
+#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with many
+   remote storage services (S3, Google Cloud, Azure, SSH, etc). DVC also
+   uses reflinks or hardlinks to avoid copy operations on checkouts; thus
+   handling large data files much more efficiently.
 
-#. *Makefile* (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph).
+#. Makefile (and analogues including ad-hoc scripts) - DVC tracks
+   dependencies (in a directed acyclic graph).
 
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
+#. `DAGsHub <https://dagshub.com/>`_ - online service to host DVC
+   projects.  It provides a useful UI around DVC repositories and integrates
+   other tools.
+
+#. `DVC Studio <https://studio.iterative.ai/>`_ - official online
+   platform for DVC projects.  It can be used to manage data and models, run
+   and track experiments, and visualize and share results.  Also, it
+   integrates with `CML (CI/CD for ML) <https://cml.dev/>` for training
+   models in the cloud or Kubernetes.
+
 
 Contributing
 ============
@@ -233,10 +252,6 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://dvc.org
    :alt: DVC logo
 
-.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
-   :target: https://travis-ci.com/iterative/dvc/branches
-   :alt: Release
-
 .. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
    :target: https://github.com/iterative/dvc/actions
    :alt: GHA Tests

dvc/_debug.py

@@ -77,15 +77,15 @@ def add_debugging_flags(parser):
     from argparse import SUPPRESS
 
     parser.add_argument(
-        "--cprofile", action="store_true", default=False, help=SUPPRESS,
+        "--cprofile", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument("--cprofile-dump", help=SUPPRESS)
     parser.add_argument(
-        "--pdb", action="store_true", default=False, help=SUPPRESS,
+        "--pdb", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument(
-        "--instrument", action="store_true", default=False, help=SUPPRESS,
+        "--instrument", action="store_true", default=False, help=SUPPRESS
     )
     parser.add_argument(
-        "--instrument-open", action="store_true", default=False, help=SUPPRESS,
+        "--instrument-open", action="store_true", default=False, help=SUPPRESS
     )

dvc/analytics.py

@@ -2,6 +2,8 @@ import json
 import logging
 import os
 
+from .env import DVC_NO_ANALYTICS
+
 logger = logging.getLogger(__name__)
 
 
@@ -32,7 +34,7 @@ def collect_and_send_report(args=None, return_code=None):
 
     with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
         json.dump(report, fobj)
-        daemon(["analytics", fobj.name])
+    daemon(["analytics", fobj.name])
 
 
 def is_enabled():
@@ -42,9 +44,12 @@ def is_enabled():
     if env2bool("DVC_TEST"):
         return False
 
-    enabled = to_bool(
-        Config(validate=False).get("core", {}).get("analytics", "true")
-    )
+    enabled = not os.getenv(DVC_NO_ANALYTICS)
+    if enabled:
+        enabled = to_bool(
+            Config(validate=False).get("core", {}).get("analytics", "true")
+        )
+
     logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
     return enabled

dvc/checkout.py

@@ -2,24 +2,26 @@ import logging
 
 from shortuuid import uuid
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.exceptions import (
     CacheLinkError,
     CheckoutError,
     ConfirmRemoveError,
     DvcException,
 )
+from dvc.ignore import DvcIgnoreFilter
 from dvc.objects import check, load
-from dvc.objects.errors import ObjectFormatError
-from dvc.objects.stage import stage
-from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
+from dvc.objects.db.slow_link_detection import (  # type: ignore[attr-defined]
     slow_link_guard,
 )
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.stage import stage
+from dvc.types import Optional
 
 logger = logging.getLogger(__name__)
 
 
-def _changed(path_info, fs, obj, cache):
+def _changed(path_info, fs, obj, cache, state=None):
     logger.trace("checking if '%s'('%s') has changed.", path_info, obj)
 
     try:
@@ -30,11 +32,15 @@ def _changed(path_info, fs, obj, cache):
         )
         return True
 
-    try:
-        actual = stage(cache, path_info, fs, obj.hash_info.name).hash_info
-    except FileNotFoundError:
-        logger.debug("'%s' doesn't exist.", path_info)
-        return True
+    actual = state.get(path_info, fs) if state else None
+    if actual is None:
+        try:
+            actual = stage(
+                cache, path_info, fs, obj.hash_info.name, dry_run=True
+            )[1].hash_info
+        except FileNotFoundError:
+            logger.debug("'%s' doesn't exist.", path_info)
+            return True
 
     if obj.hash_info != actual:
         logger.debug(
@@ -57,7 +63,9 @@ def _remove(path_info, fs, cache, force=False):
         fs.remove(path_info)
         return
 
-    current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM).hash_info
+    current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM, dry_run=True)[
+        1
+    ].hash_info
     try:
         obj = load(cache, current)
         check(cache, obj)
@@ -95,7 +103,7 @@ def _do_link(cache, from_info, to_info, link_method):
     link_method(from_info, to_info)
 
     logger.debug(
-        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info,
+        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info
     )
 
 
@@ -118,9 +126,11 @@ def _try_links(cache, from_info, to_info, link_types):
 
 
 def _link(cache, from_info, to_info):
-    assert cache.fs.isfile(from_info)
     cache.makedirs(to_info.parent)
-    _try_links(cache, from_info, to_info, cache.cache_types)
+    try:
+        _try_links(cache, from_info, to_info, cache.cache_types)
+    except FileNotFoundError as exc:
+        raise CheckoutError([to_info]) from exc
 
 
 def _cache_is_copy(cache, path_info):
@@ -132,7 +142,7 @@ def _cache_is_copy(cache, path_info):
         return True
 
     workspace_file = path_info.with_name("." + uuid())
-    test_cache_file = cache.fs.path_info / ".cache_type_test_file"
+    test_cache_file = cache.path_info / ".cache_type_test_file"
     if not cache.fs.exists(test_cache_file):
         cache.makedirs(test_cache_file.parent)
         with cache.fs.open(test_cache_file, "wb") as fobj:
@@ -148,13 +158,20 @@ def _cache_is_copy(cache, path_info):
 
 
 def _checkout_file(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
+    path_info,
+    fs,
+    obj,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    state=None,
 ):
     """The file is changed we need to checkout a new copy"""
     modified = False
     cache_info = cache.hash_to_path_info(obj.hash_info.value)
     if fs.exists(path_info):
-        if not relink and _changed(path_info, fs, obj, cache):
+        if not relink and _changed(path_info, fs, obj, cache, state=state):
             modified = True
             _remove(path_info, fs, cache, force=force)
             _link(cache, cache_info, path_info)
@@ -168,15 +185,22 @@ def _checkout_file(
         _link(cache, cache_info, path_info)
         modified = True
 
-    fs.repo.state.save(path_info, fs, obj.hash_info)
+    if state:
+        state.save(path_info, fs, obj.hash_info)
+
     if progress_callback:
         progress_callback(str(path_info))
 
     return modified
 
 
-def _remove_redundant_files(path_info, fs, obj, cache, force):
-    existing_files = set(fs.walk_files(path_info))
+def _remove_redundant_files(
+    path_info, fs, obj, cache, force, dvcignore: Optional[DvcIgnoreFilter]
+):
+    if dvcignore:
+        existing_files = set(dvcignore.walk_files(fs, path_info))
+    else:
+        existing_files = set(fs.walk_files(path_info))
 
     needed_files = {path_info.joinpath(*key) for key, _ in obj}
     redundant_files = existing_files - needed_files
@@ -187,9 +211,17 @@ def _remove_redundant_files(path_info, fs, obj, cache, force):
 
 
 def _checkout_dir(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
+    path_info,
+    fs,
+    obj,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
 ):
-    modified = False, False
+    modified = False
     # Create dir separately so that dir is created
     # even if there are no files in it
     if not fs.exists(path_info):
@@ -198,24 +230,36 @@ def _checkout_dir(
 
     logger.debug("Linking directory '%s'.", path_info)
 
+    failed = []
     for entry_key, entry_obj in obj:
-        entry_modified = _checkout_file(
-            path_info.joinpath(*entry_key),
-            fs,
-            entry_obj,
-            cache,
-            force,
-            progress_callback,
-            relink,
-        )
-        if entry_modified:
-            modified = True
+        try:
+            entry_modified = _checkout_file(
+                path_info.joinpath(*entry_key),
+                fs,
+                entry_obj,
+                cache,
+                force,
+                progress_callback,
+                relink,
+                state=None,
+            )
+            if entry_modified:
+                modified = True
+        except CheckoutError as exc:
+            failed.extend(exc.target_infos)
 
     modified = (
-        _remove_redundant_files(path_info, fs, obj, cache, force) or modified
+        _remove_redundant_files(
+            path_info, fs, obj, cache, force, dvcignore=dvcignore
+        )
+        or modified
     )
 
-    fs.repo.state.save(path_info, fs, obj.hash_info)
+    if failed:
+        raise CheckoutError(failed)
+
+    if state:
+        state.save(path_info, fs, obj.hash_info)
 
     # relink is not modified, assume it as nochange
     return modified and not relink
@@ -229,17 +273,28 @@ def _checkout(
     force=False,
     progress_callback=None,
     relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
 ):
     if not obj.hash_info.isdir:
         ret = _checkout_file(
-            path_info, fs, obj, cache, force, progress_callback, relink
+            path_info, fs, obj, cache, force, progress_callback, relink, state
         )
     else:
         ret = _checkout_dir(
-            path_info, fs, obj, cache, force, progress_callback, relink,
+            path_info,
+            fs,
+            obj,
+            cache,
+            force,
+            progress_callback,
+            relink,
+            dvcignore=dvcignore,
+            state=state,
         )
 
-    fs.repo.state.save_link(path_info, fs)
+    if state:
+        state.save_link(path_info, fs)
 
     return ret
 
@@ -253,6 +308,8 @@ def checkout(
     progress_callback=None,
     relink=False,
     quiet=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
 ):
     if path_info.scheme not in ["local", cache.fs.scheme]:
         raise NotImplementedError
@@ -268,27 +325,13 @@ def checkout(
         _remove(path_info, fs, cache, force=force)
         failed = path_info
 
-    elif not relink and not _changed(path_info, fs, obj, cache):
-        logger.trace("Data '%s' didn't change.", path_info)
+    elif not relink and not _changed(path_info, fs, obj, cache, state=state):
+        logger.trace("Data '%s' didn't change.", path_info)  # type: ignore
         skip = True
-    else:
-        try:
-            check(cache, obj)
-        except (FileNotFoundError, ObjectFormatError):
-            if not quiet:
-                logger.warning(
-                    "Cache '%s' not found. File '%s' won't be created.",
-                    obj.hash_info,
-                    path_info,
-                )
-            _remove(path_info, fs, cache, force=force)
-            failed = path_info
 
     if failed or skip:
         if progress_callback and obj:
-            progress_callback(
-                str(path_info), len(obj),
-            )
+            progress_callback(str(path_info), len(obj))
         if failed:
             raise CheckoutError([failed])
         return
@@ -296,5 +339,13 @@ def checkout(
     logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
 
     return _checkout(
-        path_info, fs, obj, cache, force, progress_callback, relink,
+        path_info,
+        fs,
+        obj,
+        cache,
+        force,
+        progress_callback,
+        relink,
+        dvcignore=dvcignore,
+        state=state,
     )

dvc/cli.py

@@ -30,6 +30,7 @@ from .command import (
     install,
     live,
     ls,
+    machine,
     metrics,
     move,
     params,
@@ -87,6 +88,7 @@ COMMANDS = [
     experiments,
     check_ignore,
     live,
+    machine,
 ]
 
 

dvc/command/add.py

@@ -117,6 +117,6 @@ def add_parser(subparsers, parent_parser):
         ),
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add.",
+        "targets", nargs="+", help="Input files/directories to add."
     ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -7,11 +7,11 @@ logger = logging.getLogger(__name__)
 
 def fix_subparsers(subparsers):
     """Workaround for bug in Python 3. See more info at:
-        https://bugs.python.org/issue16308
-        https://github.com/iterative/dvc/issues/769
+    https://bugs.python.org/issue16308
+    https://github.com/iterative/dvc/issues/769
 
-        Args:
-            subparsers: subparsers to fix.
+    Args:
+        subparsers: subparsers to fix.
     """
     subparsers.required = True
     subparsers.dest = "cmd"
@@ -33,17 +33,16 @@ class CmdBase(ABC):
 
     def __init__(self, args):
         from dvc.repo import Repo
-        from dvc.updater import Updater
 
         os.chdir(args.cd)
 
         self.repo = Repo(uninitialized=self.UNINITIALIZED)
         self.config = self.repo.config
         self.args = args
-        if self.repo.tmp_dir:
-            hardlink_lock = self.config["core"].get("hardlink_lock", False)
-            updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
-            updater.check()
+
+    def do_run(self):
+        with self.repo:
+            return self.run()
 
     @abstractmethod
     def run(self):
@@ -55,3 +54,6 @@ class CmdBaseNoRepo(CmdBase):
         self.args = args
 
         os.chdir(args.cd)
+
+    def do_run(self):
+        return self.run()

dvc/command/cache.py

@@ -1,11 +1,9 @@
 import argparse
-import logging
 
 from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 class CmdCacheDir(CmdConfig):
@@ -17,7 +15,7 @@ class CmdCacheDir(CmdConfig):
                 # Use merged config with default values
                 conf = self.config
             self._check(conf, False, "cache", "dir")
-            logger.info(conf["cache"]["dir"])
+            ui.write(conf["cache"]["dir"])
             return 0
         with self.config.edit(level=self.args.level) as conf:
             if self.args.unset:

dvc/command/check_ignore.py

@@ -1,16 +1,14 @@
 import argparse
-import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 class CmdCheckIgnore(CmdBase):
     def __init__(self, args):
         super().__init__(args)
-        self.ignore_filter = self.repo.fs.dvcignore
+        self.ignore_filter = self.repo.dvcignore
 
     def _show_results(self, result):
         if not result.match and not self.args.non_matching:
@@ -22,9 +20,9 @@ class CmdCheckIgnore(CmdBase):
                 patterns = patterns[-1:]
 
             for pattern in patterns:
-                logger.info("{}\t{}".format(pattern, result.file))
+                ui.write(pattern, result.file, sep="\t")
         else:
-            logger.info(result.file)
+            ui.write(result.file)
 
     def _check_one_file(self, target):
         result = self.ignore_filter.check_ignore(target)
@@ -122,6 +120,6 @@ def add_parser(subparsers, parent_parser):
         help="Read paths from standard input instead of providing `targets`.",
     )
     parser.add_argument(
-        "targets", nargs="*", help="File or directory paths to check",
+        "targets", nargs="*", help="File or directory paths to check"
     ).complete = completion.FILE
     parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -1,38 +1,28 @@
 import argparse
-import logging
 import operator
 
-import colorama
-
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 def log_changes(stats):
-    colors = [
-        ("modified", colorama.Fore.YELLOW,),
-        ("added", colorama.Fore.GREEN),
-        ("deleted", colorama.Fore.RED,),
-    ]
+    colors = {
+        "modified": "yellow",
+        "added": "green",
+        "deleted": "red",
+    }
 
-    for state, color in colors:
+    for state, color in colors.items():
         entries = stats.get(state)
 
         if not entries:
             continue
 
         for entry in entries:
-            logger.info(
-                "{color}{state}{nc}{spacing}{entry}".format(
-                    color=color,
-                    state=state[0].upper(),
-                    nc=colorama.Fore.RESET,
-                    spacing="\t",
-                    entry=entry,
-                )
+            ui.write(
+                f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t"
             )
 
 
@@ -58,7 +48,7 @@ class CmdCheckout(CmdBase):
             msg = get_summary(
                 sorted(stats.items(), key=operator.itemgetter(0))
             )
-            logger.info(msg or default_message)
+            ui.write(msg or default_message)
         else:
             log_changes(stats)
 

dvc/command/commit.py

@@ -71,5 +71,5 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="stages or .dvc files to commit. Optional. "
         "(Finds all DVC files in the workspace by default.)",
-    ).complete = completion.DVC_FILE
+    ).complete = completion.DVCFILES_AND_STAGE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -4,6 +4,7 @@ import logging
 import shtab
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 FILE = shtab.FILE
@@ -67,25 +68,16 @@ _dvc_compadd_exps() {
     """,
 }
 
-DVC_FILE = {
-    "bash": "_dvc_compgen_DVCFiles",
-    "zsh": "_dvc_compadd_DVCFiles",
-}
+DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
 
-STAGE = {
-    "bash": "_dvc_compgen_stages",
-    "zsh": "_dvc_compadd_stages",
-}
+STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
 
 DVCFILES_AND_STAGE = {
     "bash": "_dvc_compgen_stages_and_files",
     "zsh": "_dvc_compadd_stages_and_files",
 }
 
-EXPERIMENT = {
-    "bash": "_dvc_compgen_exps",
-    "zsh": "_dvc_compadd_exps",
-}
+EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
 
 
 class CmdCompletion(CmdBaseNoRepo):
@@ -95,7 +87,7 @@ class CmdCompletion(CmdBaseNoRepo):
         parser = get_main_parser()
         shell = self.args.shell
         script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
-        print(script)
+        ui.write(script, force=True)
         return 0
 
 

dvc/command/config.py

@@ -3,6 +3,7 @@ import logging
 import os
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -71,7 +72,9 @@ class CmdConfig(CmdBaseNoRepo):
             prefix = self._config_file_prefix(
                 self.args.show_origin, self.config, level
             )
-            logger.info("\n".join(self._format_config(conf, prefix)))
+            configs = list(self._format_config(conf, prefix))
+            if configs:
+                ui.write("\n".join(configs))
 
         return 0
 
@@ -96,7 +99,7 @@ class CmdConfig(CmdBaseNoRepo):
                 prefix = self._config_file_prefix(
                     self.args.show_origin, self.config, level
                 )
-                logger.info("{}{}".format(prefix, conf[section][opt]))
+                ui.write(prefix, conf[section][opt], sep="")
                 break
 
         return 0

dvc/command/daemon.py

@@ -67,6 +67,6 @@ def add_parser(subparsers, parent_parser):
         help=DAEMON_ANALYTICS_HELP,
     )
     daemon_analytics_parser.add_argument(
-        "target", help="Analytics file.",
+        "target", help="Analytics file."
     ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -1,9 +1,7 @@
 import argparse
-import logging
 
 from dvc.command.base import CmdBase, append_doc_link
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 def _show_ascii(G):
@@ -38,18 +36,20 @@ def _collect_targets(repo, target, outs):
         return [stage.addressing for stage, _ in pairs]
 
     targets = []
+
+    outs_trie = repo.index.outs_trie
     for stage, info in pairs:
         if not info:
             targets.extend([str(out) for out in stage.outs])
             continue
 
-        for out in repo.outs_trie.itervalues(prefix=info.parts):  # noqa: B301
+        for out in outs_trie.itervalues(prefix=info.parts):  # noqa: B301
             targets.extend(str(out))
 
     return targets
 
 
-def _transform(repo, outs):
+def _transform(index, outs):
     import networkx as nx
 
     from dvc.stage import Stage
@@ -57,7 +57,7 @@ def _transform(repo, outs):
     def _relabel(node) -> str:
         return node.addressing if isinstance(node, Stage) else str(node)
 
-    G = repo.outs_graph if outs else repo.graph
+    G = index.outs_graph if outs else index.graph
     return nx.relabel_nodes(G, _relabel, copy=True)
 
 
@@ -87,7 +87,7 @@ def _filter(G, targets, full):
 
 def _build(repo, target=None, full=False, outs=False):
     targets = _collect_targets(repo, target, outs)
-    G = _transform(repo, outs)
+    G = _transform(repo.index, outs)
     return _filter(G, targets, full)
 
 
@@ -101,7 +101,7 @@ class CmdDAG(CmdBase):
         )
 
         if self.args.dot:
-            logger.info(_show_dot(G))
+            ui.write(_show_dot(G))
         else:
             from dvc.utils.pager import pager
 

dvc/command/data_sync.py

@@ -9,10 +9,11 @@ logger = logging.getLogger(__name__)
 
 class CmdDataBase(CmdBase):
     def log_summary(self, stats):
+        from dvc.ui import ui
         from dvc.utils.humanize import get_summary
 
         default_msg = "Everything is up to date."
-        logger.info(get_summary(stats.items()) or default_msg)
+        ui.write(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
@@ -139,7 +140,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
-        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>"
     )
     pull_parser.add_argument(
         "-a",
@@ -208,7 +209,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
-        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>"
     )
     push_parser.add_argument(
         "-a",
@@ -272,10 +273,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to fetch from",
-        metavar="<name>",
+        "-r", "--remote", help="Remote storage to fetch from", metavar="<name>"
     )
     fetch_parser.add_argument(
         "-a",

dvc/command/destroy.py

@@ -1,7 +1,6 @@
 import argparse
 import logging
 
-import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)
@@ -10,6 +9,7 @@ logger = logging.getLogger(__name__)
 class CmdDestroy(CmdBase):
     def run(self):
         from dvc.exceptions import DvcException
+        from dvc.ui import ui
 
         try:
             statement = (
@@ -19,7 +19,7 @@ class CmdDestroy(CmdBase):
                 "Are you sure you want to continue?"
             )
 
-            if not self.args.force and not prompt.confirm(statement):
+            if not self.args.force and not ui.confirm(statement):
                 raise DvcException(
                     "cannot destroy without a confirmation from the user."
                     " Use `-f` to force."

dvc/command/diff.py

@@ -3,10 +3,9 @@ import json
 import logging
 import os
 
-import colorama
-
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -18,13 +17,12 @@ def _digest(checksum):
 
 
 def _show_md(diff, show_hash=False, hide_missing=False):
-    from dvc.ui import ui
-
     headers = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
     statuses = ["added", "deleted", "renamed", "modified"]
     if not hide_missing:
         statuses.append("not in cache")
+
     for status in statuses:
         entries = diff.get(status, [])
         if not entries:
@@ -44,7 +42,7 @@ def _show_md(diff, show_hash=False, hide_missing=False):
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _format(diff, hide_missing=False):
+    def _show_diff(diff, hide_missing=False):
         """
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
@@ -71,15 +69,14 @@ class CmdDiff(CmdBase):
         """
 
         colors = {
-            "added": colorama.Fore.GREEN,
-            "modified": colorama.Fore.YELLOW,
-            "deleted": colorama.Fore.RED,
-            "renamed": colorama.Fore.GREEN,
-            "not in cache": colorama.Fore.YELLOW,
+            "added": "green",
+            "modified": "yellow",
+            "deleted": "red",
+            "renamed": "green",
+            "not in cache": "yellow",
         }
 
         summary = {}
-        groups = []
 
         states = ["added", "deleted", "renamed", "modified"]
         if not hide_missing:
@@ -91,7 +88,8 @@ class CmdDiff(CmdBase):
             if not entries:
                 continue
 
-            content = []
+            header = state.capitalize()
+            ui.write(f"[{colors[state]}]{header}[/]:", styled=True)
 
             for entry in entries:
                 path = entry["path"]
@@ -99,7 +97,7 @@ class CmdDiff(CmdBase):
                     path = f"{path['old']} -> {path['new']}"
                 checksum = entry.get("hash")
                 summary[state] += 1 if not path.endswith(os.sep) else 0
-                content.append(
+                ui.write(
                     "{space}{checksum}{separator}{path}".format(
                         space="    ",
                         checksum=_digest(checksum) if checksum else "",
@@ -108,27 +106,17 @@ class CmdDiff(CmdBase):
                     )
                 )
 
-            groups.append(
-                "{color}{header}{nc}:\n{content}".format(
-                    color=colors[state],
-                    header=state.capitalize(),
-                    nc=colorama.Fore.RESET,
-                    content="\n".join(content),
-                )
-            )
+            ui.write()
 
         if not sum(summary.values()):
             return None
 
-        fmt = (
-            "files summary: {added} added, {deleted} deleted,"
-            " {renamed} renamed, {modified} modified"
+        states_summary = ", ".join(
+            f"{summary[state]} {state}"
+            for state in states
+            if summary[state] > 0
         )
-        if not hide_missing:
-            fmt += ", {not in cache} not in cache"
-        groups.append(fmt.format_map(summary))
-
-        return "\n\n".join(groups)
+        ui.write("files summary:", states_summary)
 
     def run(self):
         from dvc.exceptions import DvcException
@@ -155,13 +143,11 @@ class CmdDiff(CmdBase):
                 diff[key] = entries
 
             if self.args.show_json:
-                logger.info(json.dumps(diff))
+                ui.write(json.dumps(diff))
             elif self.args.show_md:
-                logger.info(_show_md(diff, show_hash, hide_missing))
+                _show_md(diff, show_hash, hide_missing)
             elif diff:
-                output = self._format(diff, hide_missing)
-                if output:
-                    logger.info(output)
+                self._show_diff(diff, hide_missing)
 
         except DvcException:
             logger.exception("failed to get diff")

dvc/command/experiments.py

@@ -5,18 +5,22 @@ from datetime import date, datetime
 from fnmatch import fnmatch
 from typing import TYPE_CHECKING, Dict, Iterable, Optional
 
-from funcy import constantly, iffy, lmap
+from funcy import lmap
 
-import dvc.prompt as prompt
+from dvc import prompt
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.command.metrics import DEFAULT_PRECISION
 from dvc.command.repro import CmdRepro
 from dvc.command.repro import add_arguments as add_repro_arguments
 from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.ui import ui
 from dvc.utils.flatten import flatten
+from dvc.utils.serialize import encode_exception
 
 if TYPE_CHECKING:
+    from rich.text import Text
+
     from dvc.compare import TabularData
 
 
@@ -25,6 +29,7 @@ logger = logging.getLogger(__name__)
 
 SHOW_MAX_WIDTH = 1024
 FILL_VALUE = "-"
+FILL_VALUE_ERRORED = "!"
 
 
 def _filter_name(names, label, filter_strs):
@@ -87,6 +92,7 @@ def _filter_names(
 
 def _update_names(names, items):
     for name, item in items:
+        item = item.get("data", {})
         if isinstance(item, dict):
             item = flatten(item)
             names[name].update({key: None for key in item})
@@ -97,7 +103,8 @@ def _collect_names(all_experiments, **kwargs):
     param_names = defaultdict(dict)
 
     for _, experiments in all_experiments.items():
-        for exp in experiments.values():
+        for exp_data in experiments.values():
+            exp = exp_data.get("data", {})
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
     metric_names = _filter_names(
@@ -116,6 +123,16 @@ def _collect_names(all_experiments, **kwargs):
     return metric_names, param_names
 
 
+experiment_types = {
+    "checkpoint_tip": "│ ╓",
+    "checkpoint_commit": "│ ╟",
+    "checkpoint_base": "├─╨",
+    "branch_commit": "├──",
+    "branch_base": "└──",
+    "baseline": "",
+}
+
+
 def _collect_rows(
     base_rev,
     experiments,
@@ -137,8 +154,15 @@ def _collect_rows(
         )
 
     new_checkpoint = True
-    for i, (rev, exp) in enumerate(experiments.items()):
-        queued = str(exp.get("queued") or "")
+    for i, (rev, results) in enumerate(experiments.items()):
+        exp = results.get("data", {})
+        if exp.get("running"):
+            state = "Running"
+        elif exp.get("queued"):
+            state = "Queued"
+        else:
+            state = FILL_VALUE
+        executor = exp.get("executor", FILL_VALUE)
         is_baseline = rev == "baseline"
 
         if is_baseline:
@@ -147,48 +171,57 @@ def _collect_rows(
             name_rev = rev[:7]
 
         exp_name = exp.get("name", "")
-        if is_baseline:
-            name = exp_name or name_rev
-        elif exp_name:
-            name = f"{rev[:7]} [[bold]{exp['name']}[/]]"
-        else:
-            name = name_rev
-
         tip = exp.get("checkpoint_tip")
 
         parent_rev = exp.get("checkpoint_parent", "")
-        parent_exp = experiments.get(parent_rev, {})
+        parent_exp = experiments.get(parent_rev, {}).get("data", {})
         parent_tip = parent_exp.get("checkpoint_tip")
 
         parent = ""
         if is_baseline:
-            tree = ""
+            typ = "baseline"
         elif tip:
             if tip == parent_tip:
-                tree = "│ ╓" if new_checkpoint else "│ ╟"
+                typ = (
+                    "checkpoint_tip" if new_checkpoint else "checkpoint_commit"
+                )
             elif parent_rev == base_rev:
-                tree = "├─╨"
+                typ = "checkpoint_base"
             else:
-                tree = "│ ╟"
+                typ = "checkpoint_commit"
                 parent = parent_rev[:7]
+        elif i < len(experiments) - 1:
+            typ = "branch_commit"
         else:
-            tree = "├──" if i < len(experiments) - 1 else "└──"
+            typ = "branch_base"
 
         if not is_baseline:
             new_checkpoint = not (tip and tip == parent_tip)
 
         row = [
-            name,
-            queued,
-            tree,
+            exp_name,
+            name_rev,
+            typ,
             _format_time(exp.get("timestamp")),
-            rev == "baseline",
             parent,
+            state,
+            executor,
         ]
+        fill_value = FILL_VALUE_ERRORED if results.get("error") else FILL_VALUE
         _extend_row(
-            row, metric_names, exp.get("metrics", {}).items(), precision
+            row,
+            metric_names,
+            exp.get("metrics", {}).items(),
+            precision,
+            fill_value=fill_value,
+        )
+        _extend_row(
+            row,
+            param_names,
+            exp.get("params", {}).items(),
+            precision,
+            fill_value=fill_value,
         )
-        _extend_row(row, param_names, exp.get("params", {}).items(), precision)
 
         yield row
 
@@ -251,19 +284,23 @@ def _format_time(timestamp):
     return timestamp.strftime(fmt)
 
 
-def _extend_row(row, names, items, precision):
+def _extend_row(row, names, items, precision, fill_value=FILL_VALUE):
     from rich.text import Text
 
     from dvc.compare import _format_field, with_value
 
     if not items:
-        row.extend(FILL_VALUE for keys in names.values() for _ in keys)
+        row.extend(fill_value for keys in names.values() for _ in keys)
         return
 
-    for fname, item in items:
+    for fname, data in items:
+        item = data.get("data", {})
         item = flatten(item) if isinstance(item, dict) else {fname: item}
         for name in names[fname]:
-            value = with_value(item.get(name), FILL_VALUE)
+            value = with_value(
+                item.get(name),
+                FILL_VALUE_ERRORED if data.get("error", None) else fill_value,
+            )
             # wrap field data in rich.Text, otherwise rich may
             # interpret unescaped braces from list/dict types as rich
             # markup tags
@@ -297,11 +334,12 @@ def experiments_table(
 
     headers = [
         "Experiment",
-        "queued",
-        "ident_guide",
+        "rev",
+        "typ",
         "Created",
-        "is_baseline",
         "parent",
+        "State",
+        "Executor",
     ]
     td = TabularData(
         lconcat(headers, metric_headers, param_headers), fill_value=FILL_VALUE
@@ -321,17 +359,29 @@ def experiments_table(
     return td
 
 
-def prepare_exp_id(kwargs):
+def prepare_exp_id(kwargs) -> "Text":
+    from rich.text import Text
+
     exp_name = kwargs["Experiment"]
-    ident = kwargs.get("ident_guide")
-    queued = kwargs.get("queued")
+    rev = kwargs["rev"]
+    typ = kwargs.get("typ", "baseline")
+
+    if typ == "baseline" or not exp_name:
+        text = Text(exp_name or rev)
+    else:
+        text = Text.assemble(rev, " [", (exp_name, "bold"), "]")
+
     parent = kwargs.get("parent")
-    return "{}{}{}{}".format(
-        f"{ident} " if ident else "",
-        "*" if queued else "",
-        exp_name,
-        f" ({parent})" if parent else "",
-    )
+    suff = f" ({parent})" if parent else ""
+    text.append(suff)
+
+    tree = experiment_types[typ]
+    pref = f"{tree} " if tree else ""
+    return Text(pref) + text
+
+
+def baseline_styler(typ):
+    return {"style": "bold"} if typ == "baseline" else {}
 
 
 def show_experiments(
@@ -366,11 +416,13 @@ def show_experiments(
     if no_timestamp:
         td.drop("Created")
 
-    baseline_styler = iffy(constantly({"style": "bold"}), default={})
-    row_styles = lmap(baseline_styler, td.column("is_baseline"))
-    td.drop("is_baseline")
+    for col in ("State", "Executor"):
+        if td.is_empty(col):
+            td.drop(col)
 
-    merge_headers = ["Experiment", "queued", "ident_guide", "parent"]
+    row_styles = lmap(baseline_styler, td.column("typ"))
+
+    merge_headers = ["Experiment", "rev", "typ", "parent"]
     td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers))
     td.drop(*merge_headers[1:])
 
@@ -378,12 +430,14 @@ def show_experiments(
     styles = {
         "Experiment": {"no_wrap": True, "header_style": "black on grey93"},
         "Created": {"header_style": "black on grey93"},
+        "State": {"header_style": "black on grey93"},
+        "Executor": {"header_style": "black on grey93"},
     }
     header_bg_colors = {"metrics": "cornsilk1", "params": "light_cyan1"}
     styles.update(
         {
             header: {
-                "justify": "left" if typ == "metrics" else "params",
+                "justify": "right" if typ == "metrics" else "left",
                 "header_style": f"black on {header_bg_colors[typ]}",
                 "collapse": idx != 0,
                 "no_wrap": typ == "metrics",
@@ -416,7 +470,7 @@ def _normalize_headers(names):
 def _format_json(item):
     if isinstance(item, (date, datetime)):
         return item.isoformat()
-    raise TypeError
+    return encode_exception(item)
 
 
 class CmdExperimentsShow(CmdBase):
@@ -437,7 +491,7 @@ class CmdExperimentsShow(CmdBase):
         if self.args.show_json:
             import json
 
-            logger.info(json.dumps(all_experiments, default=_format_json))
+            ui.write(json.dumps(all_experiments, default=_format_json))
         else:
             show_experiments(
                 all_experiments,
@@ -481,7 +535,7 @@ class CmdExperimentsDiff(CmdBase):
         if self.args.show_json:
             import json
 
-            logger.info(json.dumps(diff))
+            ui.write(json.dumps(diff))
         else:
             from dvc.compare import show_diff
 
@@ -489,8 +543,6 @@ class CmdExperimentsDiff(CmdBase):
             diffs = [("metrics", "Metric"), ("params", "Param")]
             for idx, (key, title) in enumerate(diffs):
                 if idx:
-                    from dvc.ui import ui
-
                     # we are printing tables even in `--quiet` mode
                     # so we should also be printing the "table" separator
                     ui.write(force=True)
@@ -524,7 +576,7 @@ class CmdExperimentsRun(CmdRepro):
                 )
 
         if self.args.reset:
-            logger.info("Any existing checkpoints will be reset and re-run.")
+            ui.write("Any existing checkpoints will be reset and re-run.")
 
         results = self.repo.experiments.run(
             name=self.args.name,
@@ -541,7 +593,7 @@ class CmdExperimentsRun(CmdRepro):
         if self.args.metrics and results:
             metrics = self.repo.metrics.show(revs=list(results))
             metrics.pop("workspace", None)
-            logger.info(show_metrics(metrics))
+            show_metrics(metrics)
 
         return 0
 
@@ -577,7 +629,7 @@ class CmdExperimentsGC(CmdRepro):
         msg += " of the current repo."
         if self.args.queued:
             msg += " Run queued experiments will be preserved."
-        if self.args.queued:
+        else:
             msg += " Run queued experiments will be removed."
 
         logger.warning(msg)
@@ -595,12 +647,13 @@ class CmdExperimentsGC(CmdRepro):
         )
 
         if removed:
-            logger.info(
-                f"Removed {removed} experiments. To remove unused cache files "
-                "use 'dvc gc'."
+            ui.write(
+                f"Removed {removed} experiments.",
+                "To remove unused cache files",
+                "use 'dvc gc'.",
             )
         else:
-            logger.info("No experiments to remove.")
+            ui.write("No experiments to remove.")
         return 0
 
 
@@ -649,15 +702,15 @@ class CmdExperimentsPush(CmdBase):
             run_cache=self.args.run_cache,
         )
 
-        logger.info(
-            "Pushed experiment '%s' to Git remote '%s'.",
-            self.args.experiment,
-            self.args.git_remote,
+        ui.write(
+            f"Pushed experiment '{self.args.experiment}'"
+            f"to Git remote '{self.args.git_remote}'."
         )
         if not self.args.push_cache:
-            logger.info(
-                "To push cached outputs for this experiment to DVC remote "
-                "storage, re-run this command without '--no-cache'."
+            ui.write(
+                "To push cached outputs",
+                "for this experiment to DVC remote storage,"
+                "re-run this command without '--no-cache'.",
             )
 
         return 0
@@ -676,15 +729,15 @@ class CmdExperimentsPull(CmdBase):
             run_cache=self.args.run_cache,
         )
 
-        logger.info(
-            "Pulled experiment '%s' from Git remote '%s'. ",
-            self.args.experiment,
-            self.args.git_remote,
+        ui.write(
+            f"Pulled experiment '{self.args.experiment}'",
+            f"from Git remote '{self.args.git_remote}'.",
         )
         if not self.args.pull_cache:
-            logger.info(
-                "To pull cached outputs for this experiment from DVC remote "
-                "storage, re-run this command without '--no-cache'."
+            ui.write(
+                "To pull cached outputs for this experiment"
+                "from DVC remote storage,"
+                "re-run this command without '--no-cache'."
             )
 
         return 0
@@ -694,7 +747,10 @@ class CmdExperimentsRemove(CmdBase):
     def run(self):
 
         self.repo.experiments.remove(
-            exp_names=self.args.experiment, queue=self.args.queue,
+            exp_names=self.args.experiment,
+            queue=self.args.queue,
+            clear_all=self.args.all,
+            remote=self.args.git_remote,
         )
 
         return 0
@@ -855,7 +911,7 @@ def add_parser(subparsers, parent_parser):
         help="Fail if this command would overwrite conflicting changes.",
     )
     experiments_apply_parser.add_argument(
-        "experiment", help="Experiment to be applied.",
+        "experiment", help="Experiment to be applied."
     ).complete = completion.EXPERIMENT
     experiments_apply_parser.set_defaults(func=CmdExperimentsApply)
 
@@ -924,9 +980,7 @@ def add_parser(subparsers, parent_parser):
     )
     experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)
 
-    EXPERIMENTS_RUN_HELP = (
-        "Reproduce complete or partial experiment pipelines."
-    )
+    EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
     experiments_run_parser = experiments_subparsers.add_parser(
         "run",
         parents=[parent_parser],
@@ -1020,10 +1074,10 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_branch_parser.add_argument(
-        "experiment", help="Experiment to be promoted.",
+        "experiment", help="Experiment to be promoted."
     )
     experiments_branch_parser.add_argument(
-        "branch", help="Git branch name to use.",
+        "branch", help="Git branch name to use."
     )
     experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)
 
@@ -1046,7 +1100,7 @@ def add_parser(subparsers, parent_parser):
         metavar="<rev>",
     )
     experiments_list_parser.add_argument(
-        "--all", action="store_true", help="List all experiments.",
+        "--all", action="store_true", help="List all experiments."
     )
     experiments_list_parser.add_argument(
         "--names-only",
@@ -1118,7 +1172,7 @@ def add_parser(subparsers, parent_parser):
         metavar="<git_remote>",
     )
     experiments_push_parser.add_argument(
-        "experiment", help="Experiment to push.", metavar="<experiment>",
+        "experiment", help="Experiment to push.", metavar="<experiment>"
     ).complete = completion.EXPERIMENT
     experiments_push_parser.set_defaults(func=CmdExperimentsPush)
 
@@ -1174,11 +1228,11 @@ def add_parser(subparsers, parent_parser):
         metavar="<git_remote>",
     )
     experiments_pull_parser.add_argument(
-        "experiment", help="Experiment to pull.", metavar="<experiment>",
+        "experiment", help="Experiment to pull.", metavar="<experiment>"
     )
     experiments_pull_parser.set_defaults(func=CmdExperimentsPull)
 
-    EXPERIMENTS_REMOVE_HELP = "Remove local experiments."
+    EXPERIMENTS_REMOVE_HELP = "Remove experiments."
     experiments_remove_parser = experiments_subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -1186,8 +1240,21 @@ def add_parser(subparsers, parent_parser):
         help=EXPERIMENTS_REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    experiments_remove_parser.add_argument(
-        "--queue", action="store_true", help="Remove all queued experiments.",
+    remove_group = experiments_remove_parser.add_mutually_exclusive_group()
+    remove_group.add_argument(
+        "--queue", action="store_true", help="Remove all queued experiments."
+    )
+    remove_group.add_argument(
+        "-A",
+        "--all",
+        action="store_true",
+        help="Remove all committed experiments.",
+    )
+    remove_group.add_argument(
+        "-g",
+        "--git-remote",
+        metavar="<git_remote>",
+        help="Name or URL of the Git remote to remove the experiment from",
     )
     experiments_remove_parser.add_argument(
         "experiment",

dvc/command/freeze.py

@@ -40,7 +40,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to freeze",
+        "targets", nargs="+", help="Stages or .dvc files to freeze"
     ).complete = completion.DVC_FILE
     freeze_parser.set_defaults(func=CmdFreeze)
 
@@ -53,6 +53,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze"
     ).complete = completion.DVC_FILE
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -2,8 +2,8 @@ import argparse
 import logging
 import os
 
-import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +44,7 @@ class CmdGC(CmdBase):
         logger.warning(msg)
 
         msg = "Are you sure you want to proceed?"
-        if not self.args.force and not prompt.confirm(msg):
+        if not self.args.force and not ui.confirm(msg):
             return 1
 
         self.repo.gc(

dvc/command/get.py

@@ -12,12 +12,13 @@ logger = logging.getLogger(__name__)
 class CmdGet(CmdBaseNoRepo):
     def _show_url(self):
         from dvc.api import get_url
+        from dvc.ui import ui
 
         try:
             url = get_url(
                 self.args.path, repo=self.args.url, rev=self.args.rev
             )
-            logger.info(url)
+            ui.write(url, force=True)
         except DvcException:
             logger.exception("failed to show URL")
             return 1
@@ -64,7 +65,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
+        "path", help="Path to a file or directory within the repository"
     ).complete = completion.FILE
     get_parser.add_argument(
         "-o",

dvc/command/get_url.py

@@ -34,7 +34,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to.",
+        "out", nargs="?", help="Destination path to put data to."
     ).complete = completion.DIR
     get_parser.add_argument(
         "-j",

dvc/command/git_hook.py

@@ -105,7 +105,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_COMMIT_HELP,
     )
     pre_commit_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     pre_commit_parser.set_defaults(func=CmdPreCommit)
 
@@ -117,7 +117,7 @@ def add_parser(subparsers, parent_parser):
         help=POST_CHECKOUT_HELP,
     )
     post_checkout_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     post_checkout_parser.set_defaults(func=CmdPostCheckout)
 
@@ -129,7 +129,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_PUSH_HELP,
     )
     pre_push_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
 
@@ -146,9 +146,7 @@ def add_parser(subparsers, parent_parser):
         help="Ancestor's version of the conflicting file.",
     )
     merge_driver_parser.add_argument(
-        "--our",
-        required=True,
-        help="Current version of the conflicting file.",
+        "--our", required=True, help="Current version of the conflicting file."
     )
     merge_driver_parser.add_argument(
         "--their",

dvc/command/imp.py

@@ -48,7 +48,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
+        "path", help="Path to a file or directory within the repository"
     ).complete = completion.FILE
     import_parser.add_argument(
         "-o",

dvc/command/imp_url.py

@@ -58,7 +58,7 @@ def add_parser(subparsers, parent_parser):
         "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to.",
+        "out", nargs="?", help="Destination path to put files to."
     ).complete = completion.DIR
     import_parser.add_argument(
         "--file",

dvc/command/init.py

@@ -12,8 +12,10 @@ logger = logging.getLogger(__name__)
 
 
 def _welcome_message():
+    from dvc.ui import ui
+
     if analytics.is_enabled():
-        logger.info(
+        ui.write(
             boxify(
                 "DVC has enabled anonymous aggregate usage analytics.\n"
                 "Read the analytics documentation (and how to opt-out) here:\n"
@@ -30,7 +32,7 @@ def _welcome_message():
         f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
     ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
 
-    logger.info(msg)
+    ui.write(msg)
 
 
 class CmdInit(CmdBaseNoRepo):
@@ -39,14 +41,14 @@ class CmdInit(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            self.repo = Repo.init(
+            with Repo.init(
                 ".",
                 no_scm=self.args.no_scm,
                 force=self.args.force,
                 subdir=self.args.subdir,
-            )
-            self.config = self.repo.config
-            _welcome_message()
+            ) as repo:
+                self.config = repo.config
+                _welcome_message()
         except InitError:
             logger.exception("failed to initiate DVC")
             return 1

dvc/command/live.py

@@ -1,11 +1,9 @@
 import argparse
-import logging
-import os
+from pathlib import Path
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, fix_subparsers
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 class CmdLive(CmdBase):
@@ -14,12 +12,14 @@ class CmdLive(CmdBase):
     def _run(self, target, revs=None):
         metrics, plots = self.repo.live.show(target=target, revs=revs)
 
-        html_path = self.args.target + ".html"
-        self.repo.plots.write_html(html_path, plots, metrics)
+        if plots:
+            html_path = Path.cwd() / (self.args.target + "_html")
+            from dvc.render.utils import render
 
-        logger.info(f"\nfile://{os.path.abspath(html_path)}")
-
-        return 0
+            index_path = render(self.repo, plots, metrics, html_path)
+            ui.write(index_path.as_uri())
+            return 0
+        return 1
 
 
 class CmdLiveShow(CmdLive):
@@ -35,7 +35,7 @@ class CmdLiveDiff(CmdLive):
 def shared_parent_parser():
     parent_parser = argparse.ArgumentParser(add_help=False)
     parent_parser.add_argument(
-        "target", help="Logs dir to produce summary from",
+        "target", help="Logs dir to produce summary from"
     ).complete = completion.DIR
     parent_parser.add_argument(
         "-o",

dvc/command/ls/__init__.py

@@ -1,11 +1,11 @@
 import argparse
 import logging
-import sys
 
 from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -37,10 +37,10 @@ class CmdList(CmdBaseNoRepo):
             if self.args.show_json:
                 import json
 
-                logger.info(json.dumps(entries))
+                ui.write(json.dumps(entries))
             elif entries:
-                entries = _prettify(entries, sys.stdout.isatty())
-                logger.info("\n".join(entries))
+                entries = _prettify(entries, with_color=True)
+                ui.write("\n".join(entries))
             return 0
         except DvcException:
             logger.exception(f"failed to list '{self.args.url}'")

dvc/command/machine.py

@@ -0,0 +1,296 @@
+import argparse
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.config import CmdConfig
+from dvc.config import ConfigError
+from dvc.ui import ui
+from dvc.utils import format_link
+
+
+class MachineDisabledError(ConfigError):
+    def __init__(self):
+        super().__init__("Machine feature is disabled")
+
+
+class CmdMachineConfig(CmdConfig):
+    def __init__(self, args):
+
+        super().__init__(args)
+        if not self.config["feature"].get("machine", False):
+            raise MachineDisabledError
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["machine"]:
+            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
+
+
+class CmdMachineAdd(CmdMachineConfig):
+    def run(self):
+        from dvc.machine import validate_name
+
+        validate_name(self.args.name)
+
+        if self.args.default:
+            ui.write(f"Setting '{self.args.name}' as a default machine.")
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["machine"] and not self.args.force:
+                raise ConfigError(
+                    "machine '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["machine"][self.args.name] = {"cloud": self.args.cloud}
+            if self.args.default:
+                conf["core"]["machine"] = self.args.name
+
+        return 0
+
+
+class CmdMachineRemove(CmdMachineConfig):
+    def run(self):
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["machine"][self.args.name]
+
+        up_to_level = self.args.level or "repo"
+        # Remove core.machine refs to this machine in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("machine") == self.args.name:
+                    del conf["core"]["machine"]
+
+            if level == up_to_level:
+                break
+
+        return 0
+
+
+class CmdMachineModify(CmdMachineConfig):
+    def run(self):
+        from dvc.config import merge
+
+        with self.config.edit(self.args.level) as conf:
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["machine"]:
+                conf["machine"][self.args.name] = {}
+            section = conf["machine"][self.args.name]
+            if self.args.unset:
+                section.pop(self.args.option, None)
+            else:
+                section[self.args.option] = self.args.value
+        return 0
+
+
+class CmdMachineDefault(CmdMachineConfig):
+    def run(self):
+        if self.args.name is None and not self.args.unset:
+            conf = self.config.read(self.args.level)
+            try:
+                print(conf["core"]["machine"])
+            except KeyError:
+                ui.write("No default machine set")
+                return 1
+        else:
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("machine", None)
+                else:
+                    merged_conf = self.config.load_config_to_level(
+                        self.args.level
+                    )
+                    if (
+                        self.args.name in conf["machine"]
+                        or self.args.name in merged_conf["machine"]
+                    ):
+                        conf["core"]["machine"] = self.args.name
+                    else:
+                        raise ConfigError(
+                            "default machine must be present in machine "
+                            "list."
+                        )
+        return 0
+
+
+class CmdMachineCreate(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.create(self.args.name)
+        return 0
+
+
+class CmdMachineDestroy(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.destroy(self.args.name)
+        return 0
+
+
+class CmdMachineSsh(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.run_shell(self.args.name)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    machine_HELP = "Set up and manage cloud machines."
+    machine_parser = subparsers.add_parser(
+        "machine",
+        parents=[parent_parser],
+        description=append_doc_link(machine_HELP, "machine"),
+        # NOTE: suppress help during development to hide command
+        # help=machine_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    machine_subparsers = machine_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc machine CMD --help` for " "command-specific help.",
+    )
+
+    fix_subparsers(machine_subparsers)
+
+    machine_ADD_HELP = "Add a new data machine."
+    machine_add_parser = machine_subparsers.add_parser(
+        "add",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_ADD_HELP, "machine/add"),
+        help=machine_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_add_parser.add_argument("name", help="Name of the machine")
+    machine_add_parser.add_argument(
+        "cloud",
+        help="Machine cloud. See full list of supported clouds at {}".format(
+            format_link(
+                "https://github.com/iterative/"
+                "terraform-provider-iterative#machine"
+            )
+        ),
+    )
+    machine_add_parser.add_argument(
+        "-d",
+        "--default",
+        action="store_true",
+        default=False,
+        help="Set as default machine.",
+    )
+    machine_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force overwriting existing configs",
+    )
+    machine_add_parser.set_defaults(func=CmdMachineAdd)
+
+    machine_DEFAULT_HELP = "Set/unset the default machine."
+    machine_default_parser = machine_subparsers.add_parser(
+        "default",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_DEFAULT_HELP, "machine/default"),
+        help=machine_DEFAULT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_default_parser.add_argument(
+        "name", nargs="?", help="Name of the machine"
+    )
+    machine_default_parser.add_argument(
+        "-u",
+        "--unset",
+        action="store_true",
+        default=False,
+        help="Unset default machine.",
+    )
+    machine_default_parser.set_defaults(func=CmdMachineDefault)
+
+    machine_MODIFY_HELP = "Modify the configuration of an machine."
+    machine_modify_parser = machine_subparsers.add_parser(
+        "modify",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_MODIFY_HELP, "machine/modify"),
+        help=machine_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_modify_parser.add_argument("name", help="Name of the machine")
+    machine_modify_parser.add_argument(
+        "option", help="Name of the option to modify."
+    )
+    machine_modify_parser.add_argument(
+        "value", nargs="?", help="(optional) Value of the option."
+    )
+    machine_modify_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    machine_modify_parser.set_defaults(func=CmdMachineModify)
+
+    machine_REMOVE_HELP = "Remove an machine."
+    machine_remove_parser = machine_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_REMOVE_HELP, "machine/remove"),
+        help=machine_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_remove_parser.add_argument(
+        "name", help="Name of the machine to remove."
+    )
+    machine_remove_parser.set_defaults(func=CmdMachineRemove)
+
+    machine_CREATE_HELP = "Create and start a machine instance."
+    machine_create_parser = machine_subparsers.add_parser(
+        "create",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_CREATE_HELP, "machine/create"),
+        help=machine_CREATE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_create_parser.add_argument(
+        "name", help="Name of the machine to create."
+    )
+    machine_create_parser.set_defaults(func=CmdMachineCreate)
+
+    machine_DESTROY_HELP = "Destroy an machine instance."
+    machine_destroy_parser = machine_subparsers.add_parser(
+        "destroy",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_DESTROY_HELP, "machine/destroy"),
+        help=machine_DESTROY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_destroy_parser.add_argument(
+        "name", help="Name of the machine instance to destroy."
+    )
+    machine_destroy_parser.set_defaults(func=CmdMachineDestroy)
+
+    machine_SSH_HELP = "Connect to a machine via SSH."
+    machine_ssh_parser = machine_subparsers.add_parser(
+        "ssh",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_SSH_HELP, "machine/ssh"),
+        help=machine_SSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_ssh_parser.add_argument(
+        "name", help="Name of the machine instance to connect to."
+    )
+    machine_ssh_parser.set_defaults(func=CmdMachineSsh)

dvc/command/metrics.py

@@ -3,7 +3,9 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.exceptions import BadMetricError, DvcException
+from dvc.exceptions import DvcException
+from dvc.ui import ui
+from dvc.utils.serialize import encode_exception
 
 logger = logging.getLogger(__name__)
 
@@ -32,17 +34,10 @@ class CmdMetricsShow(CmdMetricsBase):
         if self.args.show_json:
             import json
 
-            logger.info(json.dumps(metrics))
+            ui.write(json.dumps(metrics, default=encode_exception))
         else:
             from dvc.compare import show_metrics
 
-            # When `metrics` contains a `None` key, it means that some files
-            # specified as `targets` in `repo.metrics.show` didn't contain any
-            # metrics.
-            missing = metrics.pop(None, None)
-            if missing:
-                raise BadMetricError(missing)
-
             show_metrics(
                 metrics,
                 markdown=self.args.show_md,
@@ -73,7 +68,7 @@ class CmdMetricsDiff(CmdMetricsBase):
         if self.args.show_json:
             import json
 
-            logger.info(json.dumps(diff))
+            ui.write(json.dumps(diff))
         else:
             from dvc.compare import show_diff
 

dvc/command/move.py

@@ -37,9 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory.",
+        "src", help="Source path to a data file or directory."
     ).complete = completion.FILE
     move_parser.add_argument(
-        "dst", help="Destination path.",
+        "dst", help="Destination path."
     ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/params.py

@@ -4,6 +4,7 @@ import logging
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +28,7 @@ class CmdParamsDiff(CmdBase):
         if self.args.show_json:
             import json
 
-            logger.info(json.dumps(diff))
+            ui.write(json.dumps(diff))
         else:
             from dvc.compare import show_diff
 

dvc/command/plots.py

@@ -4,6 +4,8 @@ import logging
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.render.utils import find_vega, render
+from dvc.ui import ui
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -34,36 +36,51 @@ class CmdPlots(CmdBase):
                 return 1
 
         try:
-            plots = self._func(targets=self.args.targets, props=self._props())
+            plots_data = self._func(
+                targets=self.args.targets, props=self._props()
+            )
+
+            if not plots_data:
+                ui.error_write(
+                    "No plots were loaded, "
+                    "visualization file will not be created."
+                )
 
             if self.args.show_vega:
                 target = self.args.targets[0]
-                logger.info(plots[target])
+                plot_json = find_vega(self.repo, plots_data, target)
+                if plot_json:
+                    ui.write(plot_json)
                 return 0
 
-            rel: str = self.args.out or "plots.html"
-            path = (Path.cwd() / rel).resolve()
-            self.repo.plots.write_html(
-                path, plots=plots, html_template_path=self.args.html_template
+            rel: str = self.args.out or "dvc_plots"
+            path: Path = (Path.cwd() / rel).resolve()
+            index_path = render(
+                self.repo,
+                plots_data,
+                path=path,
+                html_template_path=self.args.html_template,
             )
 
+            assert index_path.is_absolute()
+            url = index_path.as_uri()
+            ui.write(url)
+
+            if self.args.open:
+                import webbrowser
+
+                opened = webbrowser.open(index_path)
+                if not opened:
+                    ui.error_write(
+                        "Failed to open. Please try opening it manually."
+                    )
+                    return 1
+            return 0
+
         except DvcException:
             logger.exception("")
             return 1
 
-        assert path.is_absolute()  # as_uri throws ValueError if not absolute
-        url = path.as_uri()
-        logger.info(url)
-        if self.args.open:
-            import webbrowser
-
-            opened = webbrowser.open(rel)
-            if not opened:
-                logger.error("Failed to open. Please try opening it manually.")
-                return 1
-
-        return 0
-
 
 class CmdPlotsShow(CmdPlots):
     UNINITIALIZED = True
@@ -87,7 +104,7 @@ class CmdPlotsDiff(CmdPlots):
 class CmdPlotsModify(CmdPlots):
     def run(self):
         self.repo.plots.modify(
-            self.args.target, props=self._props(), unset=self.args.unset,
+            self.args.target, props=self._props(), unset=self.args.unset
         )
         return 0
 
@@ -160,7 +177,7 @@ def add_parser(subparsers, parent_parser):
         help=argparse.SUPPRESS,
     )
     plots_diff_parser.add_argument(
-        "revisions", nargs="*", default=None, help="Git commits to plot from",
+        "revisions", nargs="*", default=None, help="Git commits to plot from"
     )
     _add_props_arguments(plots_diff_parser)
     _add_output_arguments(plots_diff_parser)
@@ -175,7 +192,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
-        "target", help="Metric file to set properties to",
+        "target", help="Metric file to set properties to"
     ).complete = completion.FILE
     _add_props_arguments(plots_modify_parser)
     plots_modify_parser.add_argument(

dvc/command/remote.py

@@ -1,12 +1,10 @@
 import argparse
-import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.ui import ui
 from dvc.utils import format_link
 
-logger = logging.getLogger(__name__)
-
 
 class CmdRemote(CmdConfig):
     def __init__(self, args):
@@ -27,7 +25,7 @@ class CmdRemoteAdd(CmdRemote):
         from dvc.config import ConfigError
 
         if self.args.default:
-            logger.info(f"Setting '{self.args.name}' as a default remote.")
+            ui.write(f"Setting '{self.args.name}' as a default remote.")
 
         with self.config.edit(self.args.level) as conf:
             if self.args.name in conf["remote"] and not self.args.force:
@@ -90,7 +88,7 @@ class CmdRemoteDefault(CmdRemote):
             try:
                 print(conf["core"]["remote"])
             except KeyError:
-                logger.info("No default remote set")
+                ui.write("No default remote set")
                 return 1
         else:
             with self.config.edit(self.args.level) as conf:
@@ -116,7 +114,7 @@ class CmdRemoteList(CmdRemote):
     def run(self):
         conf = self.config.read(self.args.level)
         for name, conf in conf["remote"].items():
-            logger.info("{}\t{}".format(name, conf["url"]))
+            ui.write(name, conf["url"], sep="\t")
         return 0
 
 

dvc/command/repro.py

@@ -1,20 +1,19 @@
 import argparse
-import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.status import CmdDataStatus
 
-logger = logging.getLogger(__name__)
-
 
 class CmdRepro(CmdBase):
     def run(self):
+        from dvc.ui import ui
+
         stages = self.repo.reproduce(**self._repro_kwargs)
         if len(stages) == 0:
-            logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+            ui.write(CmdDataStatus.UP_TO_DATE_MSG)
         else:
-            logger.info(
+            ui.write(
                 "Use `dvc push` to send your updates to " "remote storage."
             )
 

dvc/command/root.py

@@ -10,8 +10,9 @@ logger = logging.getLogger(__name__)
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
+        from dvc.ui import ui
 
-        logger.info(relpath(Repo.find_root()))
+        ui.write(relpath(Repo.find_root()))
         return 0
 
 

dvc/command/run.py

@@ -33,36 +33,17 @@ class CmdRun(CmdBase):
             )
             return 1
 
+        kwargs = vars(self.args)
+        kwargs.update(
+            {
+                "cmd": parse_cmd(self.args.cmd),
+                "fname": kwargs.pop("file"),
+                "no_exec": (self.args.no_exec or bool(self.args.checkpoints)),
+                "run_cache": not kwargs.pop("no_run_cache"),
+            }
+        )
         try:
-            self.repo.run(
-                cmd=parse_cmd(self.args.cmd),
-                outs=self.args.outs,
-                outs_no_cache=self.args.outs_no_cache,
-                metrics=self.args.metrics,
-                metrics_no_cache=self.args.metrics_no_cache,
-                plots=self.args.plots,
-                plots_no_cache=self.args.plots_no_cache,
-                live=self.args.live,
-                live_no_cache=self.args.live_no_cache,
-                live_no_summary=self.args.live_no_summary,
-                live_no_html=self.args.live_no_html,
-                deps=self.args.deps,
-                params=self.args.params,
-                fname=self.args.file,
-                wdir=self.args.wdir,
-                no_exec=(self.args.no_exec or bool(self.args.checkpoints)),
-                force=self.args.force,
-                run_cache=not self.args.no_run_cache,
-                no_commit=self.args.no_commit,
-                outs_persist=self.args.outs_persist,
-                outs_persist_no_cache=self.args.outs_persist_no_cache,
-                checkpoints=self.args.checkpoints,
-                always_changed=self.args.always_changed,
-                name=self.args.name,
-                single_stage=self.args.single_stage,
-                external=self.args.external,
-                desc=self.args.desc,
-            )
+            self.repo.run(**kwargs)
         except DvcException:
             logger.exception("")
             return 1
@@ -83,11 +64,9 @@ def add_parser(subparsers, parent_parser):
         help=RUN_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    run_parser.add_argument("-n", "--name", help="Stage name.")
     run_parser.add_argument(
-        "-n", "--name", help="Stage name.",
-    )
-    run_parser.add_argument(
-        "--file", metavar="<filename>", help=argparse.SUPPRESS,
+        "--file", metavar="<filename>", help=argparse.SUPPRESS
     )
     run_parser.add_argument(
         "--no-exec",

dvc/command/stage.py

@@ -9,7 +9,7 @@ from dvc.utils.cli_parse import parse_params
 from dvc.utils.humanize import truncate_text
 
 if TYPE_CHECKING:
-    from dvc.output.base import BaseOutput
+    from dvc.output import Output
     from dvc.stage import Stage
 
 logger = logging.getLogger(__name__)
@@ -18,7 +18,7 @@ MAX_TEXT_LENGTH = 80
 
 
 def generate_description(stage: "Stage") -> str:
-    def part_desc(outs: Iterable["BaseOutput"]) -> str:
+    def part_desc(outs: Iterable["Output"]) -> str:
         return ", ".join(out.def_path for out in outs)
 
     if not stage.deps and not stage.outs:
@@ -27,7 +27,7 @@ def generate_description(stage: "Stage") -> str:
     if not stage.outs and stage.deps:
         return "Depends on " + part_desc(stage.deps)
 
-    def is_plot_or_metric(out: "BaseOutput"):
+    def is_plot_or_metric(out: "Output"):
         return bool(out.plot) or bool(out.metric)
 
     desc: List[str] = []
@@ -66,7 +66,7 @@ def prepare_stages_data(
 class CmdStageList(CmdBase):
     def _get_stages(self) -> Iterable["Stage"]:
         if self.args.all:
-            stages: List["Stage"] = self.repo.stages  # type: ignore
+            stages: List["Stage"] = self.repo.index.stages  # type: ignore
             logger.trace(  # type: ignore[attr-defined]
                 "%d no. of stages found", len(stages)
             )
@@ -75,9 +75,7 @@ class CmdStageList(CmdBase):
         # removing duplicates while maintaining order
         collected = chain.from_iterable(
             self.repo.stage.collect(
-                target=target,
-                recursive=self.args.recursive,
-                accept_group=True,
+                target=target, recursive=self.args.recursive, accept_group=True
             )
             for target in self.args.targets
         )
@@ -115,8 +113,11 @@ def parse_cmd(commands: List[str]) -> str:
     """
 
     def quote_argument(arg: str):
-        should_quote = " " in arg and '"' not in arg
-        return f'"{arg}"' if should_quote else arg
+        if not arg:
+            return '""'
+        if " " in arg and '"' not in arg:
+            return f'"{arg}"'
+        return arg
 
     if len(commands) < 2:
         return " ".join(commands)
@@ -201,7 +202,7 @@ def _add_common_args(parser):
         metavar="<path>",
     )
     parser.add_argument(
-        "--live", help="Declare output as dvclive.", metavar="<path>",
+        "--live", help="Declare output as dvclive.", metavar="<path>"
     )
     parser.add_argument(
         "--live-no-cache",

dvc/command/status.py

@@ -2,6 +2,7 @@ import logging
 
 from dvc.command.data_sync import CmdDataBase
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -26,7 +27,7 @@ class CmdDataStatus(CmdDataBase):
         ind = indent * self.STATUS_INDENT
 
         if isinstance(status, str):
-            logger.info(f"{ind}{status}")
+            ui.write(f"{ind}{status}")
             return
 
         if isinstance(status, list):
@@ -38,14 +39,13 @@ class CmdDataStatus(CmdDataBase):
 
         for key, value in status.items():
             if isinstance(value, str):
-                logger.info("{}{}{}".format(ind, self._normalize(value), key))
+                ui.write(f"{ind}{self._normalize(value)}{key}")
             elif value:
-                logger.info(f"{ind}{key}:")
+                ui.write(f"{ind}{key}:")
                 self._show(value, indent + 1)
 
     def run(self):
         from dvc.repo import lock_repo
-        from dvc.ui import ui
 
         indent = 1 if self.args.cloud else 0
 
@@ -72,7 +72,7 @@ class CmdDataStatus(CmdDataBase):
             if self.args.show_json:
                 import json
 
-                logger.info(json.dumps(st))
+                ui.write(json.dumps(st))
                 return 0
 
             if st:
@@ -80,7 +80,7 @@ class CmdDataStatus(CmdDataBase):
                 return 0
 
             # additional hints for the user
-            if not self.repo.stages:
+            if not self.repo.index.stages:
                 ui.write(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(

dvc/command/unprotect.py

@@ -33,6 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect.",
+        "targets", nargs="+", help="Data files/directories to unprotect."
     ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -36,7 +36,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help=".dvc files to update.",
+        "targets", nargs="+", help=".dvc files to update."
     ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",

dvc/command/version.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -9,9 +10,12 @@ logger = logging.getLogger(__name__)
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
         from dvc.info import get_dvc_info
+        from dvc.updater import notify_updates
 
         dvc_info = get_dvc_info()
-        logger.info(dvc_info)
+        ui.write(dvc_info, force=True)
+
+        notify_updates()
         return 0
 
 

dvc/compare.py

@@ -1,4 +1,4 @@
-from collections.abc import Mapping
+from collections import abc
 from itertools import chain, repeat, zip_longest
 from operator import itemgetter
 from typing import (
@@ -9,6 +9,7 @@ from typing import (
     Iterable,
     Iterator,
     List,
+    Mapping,
     MutableSequence,
     Sequence,
     Tuple,
@@ -133,6 +134,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         self.drop(*(set(self._keys) - set(col_names)))
         self._keys = list(col_names)
 
+    def is_empty(self, col_name: str) -> bool:
+        col = self.column(col_name)
+        return not any(item != self._fill_value for item in col)
+
     def to_csv(self) -> str:
         import csv
         from io import StringIO
@@ -149,7 +154,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
         self._columns[name] = Column([self._fill_value] * len(self))
         self._keys.append(name)
 
-    def row_from_dict(self, d: Dict[str, "CellT"]) -> None:
+    def row_from_dict(self, d: Mapping[str, "CellT"]) -> None:
         keys = self.keys()
         for key in d:
             if key not in keys:
@@ -165,7 +170,9 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
 
         ui.table(self, headers=self.keys(), **kwargs)
 
-    def as_dict(self, cols: Iterable[str] = None) -> Iterable[Dict[str, str]]:
+    def as_dict(
+        self, cols: Iterable[str] = None
+    ) -> Iterable[Dict[str, "CellT"]]:
         keys = self.keys() if cols is None else set(cols)
         return [
             {k: self._columns[k][i] for k in keys} for i in range(len(self))
@@ -183,7 +190,7 @@ def _format_field(
         if isinstance(_val, float) and precision:
             func = round if round_digits else _normalize_float
             return func(_val, precision)
-        if isinstance(_val, Mapping):
+        if isinstance(_val, abc.Mapping):
             return {k: _format(v) for k, v in _val.items()}
         if isinstance(_val, list):
             return [_format(x) for x in _val]
@@ -274,11 +281,9 @@ def metrics_table(
     td = TabularData(["Revision", "Path"], fill_value="-")
 
     for branch, val in metrics.items():
-        for fname, metric in val.items():
-            row_data: Dict[str, str] = {
-                "Revision": branch,
-                "Path": fname,
-            }
+        for fname, metric in val.get("data", {}).items():
+            row_data: Dict[str, str] = {"Revision": branch, "Path": fname}
+            metric = metric.get("data", {})
             flattened = (
                 flatten(format_dict(metric))
                 if isinstance(metric, dict)

dvc/config.py

@@ -20,7 +20,27 @@ class ConfigError(DvcException):
         super().__init__(f"config file error: {msg}")
 
 
-class NoRemoteError(ConfigError):
+class RemoteConfigError(ConfigError):
+    pass
+
+
+class NoRemoteError(RemoteConfigError):
+    pass
+
+
+class RemoteNotFoundError(RemoteConfigError):
+    pass
+
+
+class MachineConfigError(ConfigError):
+    pass
+
+
+class NoMachineError(MachineConfigError):
+    pass
+
+
+class MachineNotFoundError(MachineConfigError):
     pass
 
 
@@ -63,7 +83,7 @@ class Config(dict):
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True, fs=None, config=None,
+        self, dvc_dir=None, validate=True, fs=None, config=None
     ):  # pylint: disable=super-init-not-called
         from dvc.fs.local import LocalFileSystem
 
@@ -79,7 +99,7 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.wfs = LocalFileSystem(None, {"url": self.dvc_dir})
+        self.wfs = LocalFileSystem(url=self.dvc_dir)
         self.fs = fs or self.wfs
 
         self.load(validate=validate, config=config)
@@ -154,12 +174,12 @@ class Config(dict):
         filename = self.files[level]
         fs = self._get_fs(level)
 
-        if fs.exists(filename, use_dvcignore=False):
+        if fs.exists(filename):
             with fs.open(filename) as fobj:
                 conf_obj = ConfigObj(fobj)
         else:
             conf_obj = ConfigObj()
-        return _parse_remotes(_lower_keys(conf_obj.dict()))
+        return _parse_named(_lower_keys(conf_obj.dict()))
 
     def _save_config(self, level, conf_dict):
         from configobj import ConfigObj
@@ -171,7 +191,7 @@ class Config(dict):
 
         fs.makedirs(os.path.dirname(filename))
 
-        config = ConfigObj(_pack_remotes(conf_dict))
+        config = ConfigObj(_pack_named(conf_dict))
         with fs.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
@@ -243,6 +263,7 @@ class Config(dict):
                     "key_path": func,
                 }
             },
+            "machine": {str: {"startup_script": func}},
         }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
 
@@ -292,27 +313,29 @@ class Config(dict):
             raise ConfigError(str(exc)) from None
 
 
-def _parse_remotes(conf):
-    result = {"remote": {}}
+def _parse_named(conf):
+    result = {"remote": {}, "machine": {}}
 
     for section, val in conf.items():
-        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
-        if name:
-            result["remote"][name] = val
+        match = re_find(r'^\s*(remote|machine)\s*"(.*)"\s*$', section)
+        if match:
+            key, name = match
+            result[key][name] = val
         else:
             result[section] = val
 
     return result
 
 
-def _pack_remotes(conf):
+def _pack_named(conf):
     # Drop empty sections
     result = compact(conf)
 
     # Transform remote.name -> 'remote "name"'
-    for name, val in conf["remote"].items():
-        result[f'remote "{name}"'] = val
-    result.pop("remote", None)
+    for key in ("remote", "machine"):
+        for name, val in conf[key].items():
+            result[f'{key} "{name}"'] = val
+        result.pop(key, None)
 
     return result
 

dvc/config_schema.py

@@ -93,7 +93,7 @@ HTTP_COMMON = {
     "user": str,
     "password": str,
     "ask_password": Bool,
-    "ssl_verify": Bool,
+    "ssl_verify": Any(Bool, str),
     "method": str,
 }
 WEBDAV_COMMON = {
@@ -104,6 +104,7 @@ WEBDAV_COMMON = {
     "cert_path": str,
     "key_path": str,
     "timeout": Coerce(int),
+    "ssl_verify": Any(Bool, str),
 }
 
 SCHEMA = {
@@ -117,6 +118,7 @@ SCHEMA = {
         Optional("autostage", default=False): Bool,
         Optional("experiments"): Bool,  # obsoleted
         Optional("check_update", default=True): Bool,
+        "machine": Lower,
     },
     "cache": {
         "local": str,
@@ -145,7 +147,7 @@ SCHEMA = {
                     "session_token": str,
                     Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
-                    Optional("ssl_verify", default=True): Bool,
+                    "ssl_verify": Any(Bool, str),
                     "sse": str,
                     "sse_kms_key_id": str,
                     "acl": str,
@@ -153,6 +155,7 @@ SCHEMA = {
                     "grant_read_acp": str,
                     "grant_write_acp": str,
                     "grant_full_control": str,
+                    "cache_regions": bool,
                     **REMOTE_COMMON,
                 },
                 "gs": {
@@ -188,6 +191,11 @@ SCHEMA = {
                     "tenant_id": str,
                     "client_id": str,
                     "client_secret": str,
+                    "allow_anonymous_login": Bool,
+                    "exclude_environment_credential": Bool,
+                    "exclude_visual_studio_code_credential": Bool,
+                    "exclude_shared_token_cache_credential": Bool,
+                    "exclude_managed_identity_credential": Bool,
                     **REMOTE_COMMON,
                 },
                 "oss": {
@@ -218,10 +226,18 @@ SCHEMA = {
         "row_limit": All(Coerce(int), Range(1)),  # obsoleted
         "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
     },
+    "machine": {
+        str: {
+            "cloud": All(Lower, Choices("aws", "azure")),
+            "instance_type": Lower,
+            "startup_script": str,
+        },
+    },
     # section for experimental features
     "feature": {
+        Optional("machine", default=False): Bool,
         # enabled by default. It's of no use, kept for backward compatibility.
-        Optional("parametrization", default=True): Bool
+        Optional("parametrization", default=True): Bool,
     },
     "plots": {"html_template": str},
 }

dvc/daemon.py

@@ -12,29 +12,34 @@ from dvc.utils import fix_env, is_binary
 
 logger = logging.getLogger(__name__)
 
-CREATE_NEW_PROCESS_GROUP = 0x00000200
-DETACHED_PROCESS = 0x00000008
+CREATE_NO_WINDOW = 0x08000000  # only available since Python 3.7 in subprocess.
 
 
 def _popen(cmd, **kwargs):
     prefix = [sys.executable]
     if not is_binary():
-        prefix += [sys.argv[0]]
-
+        main_entrypoint = os.path.join(
+            os.path.abspath(os.path.dirname(__file__)), "__main__.py"
+        )
+        prefix += [main_entrypoint]
     return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
 
 
 def _spawn_windows(cmd, env):
-    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
+    from subprocess import (
+        CREATE_NEW_PROCESS_GROUP,
+        STARTF_USESHOWWINDOW,
+        STARTUPINFO,
+    )
 
-    creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
+    # https://stackoverflow.com/a/7006424
+    # https://bugs.python.org/issue41619
+    creationflags = CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    _popen(
-        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
-    ).communicate()
+    _popen(cmd, env=env, creationflags=creationflags, startupinfo=startupinfo)
 
 
 def _spawn_posix(cmd, env):

dvc/dagascii.py

@@ -163,7 +163,7 @@ class AsciiCanvas:
         self.point(x0 + width, y0 + height, "+")
 
 
-def _build_sugiyama_layout(vertexes, edges):
+def _build_sugiyama_layout(vertices, edges):
     #
     # Just a reminder about naming conventions:
     # +------------X
@@ -174,17 +174,17 @@ def _build_sugiyama_layout(vertexes, edges):
     # Y
     #
 
-    vertexes = {v: Vertex(f" {v} ") for v in vertexes}
+    vertices = {v: Vertex(f" {v} ") for v in vertices}
     # NOTE: reverting edges to correctly orientate the graph
-    edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
-    vertexes = vertexes.values()
-    graph = Graph(vertexes, edges)
+    edges = [Edge(vertices[e], vertices[s]) for s, e in edges]
+    vertices = vertices.values()
+    graph = Graph(vertices, edges)
 
-    for vertex in vertexes:
+    for vertex in vertices:
         vertex.view = VertexViewer(vertex.data)
 
     # NOTE: determine min box length to create the best layout
-    minw = min(v.view.w for v in vertexes)
+    minw = min(v.view.w for v in vertices)
 
     for edge in edges:
         edge.view = EdgeViewer()
@@ -204,11 +204,11 @@ def _build_sugiyama_layout(vertexes, edges):
     return sug
 
 
-def draw(vertexes, edges):
+def draw(vertices, edges):
     """Build a DAG and draw it in ASCII.
 
     Args:
-        vertexes (list): list of graph vertexes.
+        vertices (list): list of graph vertices.
         edges (list): list of graph edges.
     """
     # pylint: disable=too-many-locals
@@ -217,7 +217,7 @@ def draw(vertexes, edges):
     Xs = []  # pylint: disable=invalid-name
     Ys = []  # pylint: disable=invalid-name
 
-    sug = _build_sugiyama_layout(vertexes, edges)
+    sug = _build_sugiyama_layout(vertices, edges)
 
     for vertex in sug.g.sV:
         # NOTE: moving boxes w/2 to the left

dvc/data_cloud.py

@@ -1,6 +1,13 @@
 """Manages dvc remotes that user can use with push/pull/status commands."""
 
 import logging
+from typing import TYPE_CHECKING, Iterable, Optional
+
+from dvc.objects.db import get_index
+
+if TYPE_CHECKING:
+    from dvc.hash_info import HashInfo
+    from dvc.objects.db.base import ObjectDB
 
 logger = logging.getLogger(__name__)
 
@@ -19,14 +26,18 @@ class DataCloud:
     def __init__(self, repo):
         self.repo = repo
 
-    def get_remote(self, name=None, command="<command>"):
+    def get_remote_odb(
+        self,
+        name: Optional[str] = None,
+        command: str = "<command>",
+    ) -> "ObjectDB":
         from dvc.config import NoRemoteError
 
         if not name:
             name = self.repo.config["core"].get("remote")
 
         if name:
-            return self._init_remote(name)
+            return self._init_odb(name)
 
         if bool(self.repo.config["remote"]):
             error_msg = (
@@ -43,85 +54,107 @@ class DataCloud:
 
         raise NoRemoteError(error_msg)
 
-    def _init_remote(self, name):
-        from dvc.remote import get_remote
+    def _init_odb(self, name):
+        from dvc.fs import get_cloud_fs
+        from dvc.objects.db import get_odb
 
-        return get_remote(self.repo, name=name)
+        cls, config, path_info = get_cloud_fs(self.repo, name=name)
+        config["tmp_dir"] = self.repo.tmp_dir
+        return get_odb(cls(**config), path_info, **config)
 
     def push(
-        self, cache, jobs=None, remote=None, show_checksums=False,
+        self,
+        objs: Iterable["HashInfo"],
+        jobs: Optional[int] = None,
+        remote: Optional[str] = None,
+        odb: Optional["ObjectDB"] = None,
     ):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            cache (NamedCache): named checksums to push to the cloud.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to push to.
+            objs: objects to push to the cloud.
+            jobs: number of jobs that can be running simultaneously.
+            remote: optional name of remote to push to.
                 By default remote from core.remote config option is used.
-            show_checksums (bool): show checksums instead of file names in
-                information messages.
+            odb: optional ODB to push to. Overrides remote.
         """
-        remote = self.get_remote(remote, "push")
+        from dvc.objects.transfer import transfer
 
-        return remote.push(
+        if not odb:
+            odb = self.get_remote_odb(remote, "push")
+        return transfer(
             self.repo.odb.local,
-            cache,
+            odb,
+            objs,
             jobs=jobs,
-            show_checksums=show_checksums,
+            dest_index=get_index(odb),
+            cache_odb=self.repo.odb.local,
         )
 
     def pull(
-        self, cache, jobs=None, remote=None, show_checksums=False,
+        self,
+        objs: Iterable["HashInfo"],
+        jobs: Optional[int] = None,
+        remote: Optional[str] = None,
+        odb: Optional["ObjectDB"] = None,
     ):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            cache (NamedCache): named checksums to pull from the cloud.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to pull from.
+            objs: objects to pull from the cloud.
+            jobs: number of jobs that can be running simultaneously.
+            remote: optional name of remote to pull from.
                 By default remote from core.remote config option is used.
-            show_checksums (bool): show checksums instead of file names in
-                information messages.
+            odb: optional ODB to pull from. Overrides remote.
         """
-        remote = self.get_remote(remote, "pull")
+        from dvc.objects.transfer import transfer
 
-        return remote.pull(
+        if not odb:
+            odb = self.get_remote_odb(remote, "pull")
+        return transfer(
+            odb,
             self.repo.odb.local,
-            cache,
+            objs,
             jobs=jobs,
-            show_checksums=show_checksums,
+            src_index=get_index(odb),
+            cache_odb=self.repo.odb.local,
+            verify=odb.verify,
         )
 
     def status(
         self,
-        cache,
-        jobs=None,
-        remote=None,
-        show_checksums=False,
-        log_missing=True,
+        objs: Iterable["HashInfo"],
+        jobs: Optional[int] = None,
+        remote: Optional[str] = None,
+        odb: Optional["ObjectDB"] = None,
+        log_missing: bool = True,
     ):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            cache (NamedCache): named checksums to check status for.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to compare
+            objs: objects to check status for.
+            jobs: number of jobs that can be running simultaneously.
+            remote: optional remote to compare
                 cache to. By default remote from core.remote config option
                 is used.
-            show_checksums (bool): show checksums instead of file names in
-                information messages.
-            log_missing (bool): log warning messages if file doesn't exist
+            odb: optional ODB to check status from. Overrides remote.
+            log_missing: log warning messages if file doesn't exist
                 neither in cache, neither in cloud.
         """
-        remote = self.get_remote(remote, "status")
-        return remote.status(
+        from dvc.objects.status import compare_status
+
+        if not odb:
+            odb = self.get_remote_odb(remote, "status")
+        return compare_status(
             self.repo.odb.local,
-            cache,
+            odb,
+            objs,
             jobs=jobs,
-            show_checksums=show_checksums,
             log_missing=log_missing,
+            dest_index=get_index(odb),
+            cache_odb=self.repo.odb.local,
         )
 
     def get_url_for(self, remote, checksum):
-        remote = self.get_remote(remote)
-        return str(remote.odb.hash_to_path_info(checksum))
+        remote_odb = self.get_remote_odb(remote)
+        return str(remote_odb.hash_to_path_info(checksum))

dvc/dependency/__init__.py

@@ -1,58 +1,24 @@
 from collections import defaultdict
-from urllib.parse import urlparse
+from typing import Any, Mapping
 
-import dvc.output as output
-from dvc.dependency.azure import AzureDependency
-from dvc.dependency.gs import GSDependency
-from dvc.dependency.hdfs import HDFSDependency
-from dvc.dependency.http import HTTPDependency
-from dvc.dependency.https import HTTPSDependency
-from dvc.dependency.local import LocalDependency
-from dvc.dependency.param import ParamsDependency
-from dvc.dependency.s3 import S3Dependency
-from dvc.dependency.ssh import SSHDependency
-from dvc.dependency.webdav import WebDAVDependency
-from dvc.dependency.webdavs import WebDAVSDependency
-from dvc.dependency.webhdfs import WebHDFSDependency
-from dvc.output.base import BaseOutput
-from dvc.scheme import Schemes
+from dvc.output import ARTIFACT_SCHEMA, Output
 
-from ..fs import get_cloud_fs
+from .base import Dependency
+from .param import ParamsDependency
 from .repo import RepoDependency
 
-DEP_MAP = {
-    Schemes.LOCAL: LocalDependency,
-    Schemes.SSH: SSHDependency,
-    Schemes.S3: S3Dependency,
-    Schemes.AZURE: AzureDependency,
-    Schemes.GS: GSDependency,
-    Schemes.HDFS: HDFSDependency,
-    Schemes.HTTP: HTTPDependency,
-    Schemes.HTTPS: HTTPSDependency,
-    Schemes.WEBDAV: WebDAVDependency,
-    Schemes.WEBDAVS: WebDAVSDependency,
-    Schemes.WEBHDFS: WebHDFSDependency,
-}
-
-
 # NOTE: schema for dependencies is basically the same as for outputs, but
 # without output-specific entries like 'cache' (whether or not output is
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metrics file and how to parse it, see `-M` flag for `dvc run`).
-SCHEMA = output.SCHEMA.copy()
-del SCHEMA[BaseOutput.PARAM_CACHE]
-del SCHEMA[BaseOutput.PARAM_METRIC]
-del SCHEMA[BaseOutput.PARAM_DESC]
-SCHEMA.update(RepoDependency.REPO_SCHEMA)
-SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
+SCHEMA: Mapping[str, Any] = {
+    **ARTIFACT_SCHEMA,
+    **RepoDependency.REPO_SCHEMA,
+    **ParamsDependency.PARAM_SCHEMA,
+}
 
 
 def _get(stage, p, info):
-    parsed = urlparse(p) if p else None
-    if parsed and parsed.scheme == "remote":
-        fs = get_cloud_fs(stage.repo, name=parsed.netloc)
-        return DEP_MAP[fs.scheme](stage, p, info, fs=fs)
-
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)
         return RepoDependency(repo, stage, p, info)
@@ -61,14 +27,13 @@ def _get(stage, p, info):
         params = info.pop(ParamsDependency.PARAM_PARAMS)
         return ParamsDependency(stage, p, params)
 
-    dep_cls = DEP_MAP.get(parsed.scheme, LocalDependency)
-    return dep_cls(stage, p, info)
+    return Dependency(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(BaseOutput.PARAM_PATH, None)
+        p = d.pop(Output.PARAM_PATH, None)
         ret.append(_get(stage, p, d))
     return ret
 

dvc/dependency/azure.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.azure import AzureFileSystem
-
-
-class AzureDependency(BaseDependency, BaseOutput):
-    FS_CLS = AzureFileSystem

dvc/dependency/base.py

@@ -1,6 +1,7 @@
 from typing import Type
 
 from dvc.exceptions import DvcException
+from dvc.output import Output
 
 
 class DependencyDoesNotExistError(DvcException):
@@ -20,7 +21,7 @@ class DependencyIsStageFileError(DvcException):
         super().__init__(f"DVC file '{path}' cannot be a dependency.")
 
 
-class BaseDependency:
+class Dependency(Output):
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError  # type: Type[DvcException]

dvc/dependency/gs.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.gs import GSFileSystem
-
-
-class GSDependency(BaseDependency, BaseOutput):
-    FS_CLS = GSFileSystem

dvc/dependency/hdfs.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.hdfs import HDFSOutput
-
-
-class HDFSDependency(BaseDependency, HDFSOutput):
-    pass

dvc/dependency/http.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.http import HTTPFileSystem
-
-
-class HTTPDependency(BaseDependency, BaseOutput):
-    FS_CLS = HTTPFileSystem

dvc/dependency/https.py

@@ -1,6 +0,0 @@
-from ..fs.https import HTTPSFileSystem
-from .http import HTTPDependency
-
-
-class HTTPSDependency(HTTPDependency):
-    FS_CLS = HTTPSFileSystem

dvc/dependency/local.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.local import LocalOutput
-
-
-class LocalDependency(BaseDependency, LocalOutput):
-    pass

dvc/dependency/param.py

@@ -4,11 +4,12 @@ from collections import defaultdict
 import dpath.util
 from voluptuous import Any
 
-from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
 from dvc.hash_info import HashInfo
 from dvc.utils.serialize import LOADERS, ParseError
 
+from .base import Dependency
+
 
 class MissingParamsError(DvcException):
     pass
@@ -18,7 +19,7 @@ class BadParamFileError(DvcException):
     pass
 
 
-class ParamsDependency(LocalDependency):
+class ParamsDependency(Dependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
@@ -39,7 +40,6 @@ class ParamsDependency(LocalDependency):
             path
             or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
             info=info,
-            fs=stage.repo.fs,
         )
 
     def dumpd(self):
@@ -98,7 +98,7 @@ class ParamsDependency(LocalDependency):
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
-    def read_params_d(self):
+    def read_params_d(self, **kwargs):
         config = self._read()
 
         ret = {}
@@ -128,7 +128,7 @@ class ParamsDependency(LocalDependency):
         if missing_params:
             raise MissingParamsError(
                 "Parameters '{}' are missing from '{}'.".format(
-                    ", ".join(missing_params), self,
+                    ", ".join(missing_params), self
                 )
             )
 

dvc/dependency/repo.py

@@ -1,11 +1,21 @@
+import os
+from collections import defaultdict
+from copy import copy
+from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
+
 from voluptuous import Required
 
 from dvc.path_info import PathInfo
 
-from .local import LocalDependency
+from .base import Dependency
+
+if TYPE_CHECKING:
+    from dvc.hash_info import HashInfo
+    from dvc.objects.db.base import ObjectDB
+    from dvc.objects.file import HashFile
 
 
-class RepoDependency(LocalDependency):
+class RepoDependency(Dependency):
     PARAM_REPO = "repo"
     PARAM_URL = "url"
     PARAM_REV = "rev"
@@ -21,47 +31,22 @@ class RepoDependency(LocalDependency):
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
+        self._staged_objs: Dict[str, "HashFile"] = {}
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, fs, path):
+    def _parse_path(self, fs, path_info):
         return None
 
     @property
     def is_in_repo(self):
         return False
 
-    @property
-    def repo_pair(self):
-        d = self.def_repo
-        rev = d.get(self.PARAM_REV_LOCK) or d.get(self.PARAM_REV)
-        return d[self.PARAM_URL], rev
-
     def __str__(self):
-        return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
-
-    def _make_repo(self, *, locked=True, **kwargs):
-        from dvc.external_repo import external_repo
-
-        d = self.def_repo
-        rev = (d.get("rev_lock") if locked else None) or d.get("rev")
-        return external_repo(d["url"], rev=rev, **kwargs)
-
-    def _get_hash(self, locked=True):
-        from dvc.objects.stage import stage
-
-        with self._make_repo(locked=locked) as repo:
-            path_info = PathInfo(repo.root_dir) / self.def_path
-            return stage(
-                self.repo.odb.local,
-                path_info,
-                repo.repo_fs,
-                self.repo.odb.local.fs.PARAM_CHECKSUM,
-                follow_subrepos=False,
-            ).hash_info
+        return f"{self.def_path} ({self.def_repo[self.PARAM_URL]})"
 
     def workspace_status(self):
-        current = self._get_hash(locked=True)
-        updated = self._get_hash(locked=False)
+        current = self.get_obj(locked=True).hash_info
+        updated = self.get_obj(locked=False).hash_info
 
         if current != updated:
             return {str(self): "update available"}
@@ -79,37 +64,23 @@ class RepoDependency(LocalDependency):
 
     def download(self, to, jobs=None):
         from dvc.checkout import checkout
-        from dvc.config import NoRemoteError
-        from dvc.exceptions import NoOutputOrStageError
-        from dvc.objects import save
-        from dvc.objects.stage import stage
 
-        odb = self.repo.odb.local
+        for odb, objs in self.get_used_objs().items():
+            self.repo.cloud.pull(objs, jobs=jobs, odb=odb)
 
-        with self._make_repo(cache_dir=odb.cache_dir) as repo:
-            if self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
-            path_info = PathInfo(repo.root_dir) / self.def_path
-            try:
-                repo.fetch([path_info.fspath], jobs=jobs, recursive=True)
-            except (NoOutputOrStageError, NoRemoteError):
-                pass
-            obj = stage(
-                odb,
-                path_info,
-                repo.repo_fs,
-                odb.fs.PARAM_CHECKSUM,
-                jobs=jobs,
-                follow_subrepos=False,
-            )
-            save(odb, obj, jobs=jobs)
-
-        checkout(to.path_info, to.fs, obj, odb)
+        obj = self.get_obj()
+        checkout(
+            to.path_info,
+            to.fs,
+            obj,
+            self.repo.odb.local,
+            dvcignore=None,
+            state=self.repo.state,
+        )
 
     def update(self, rev=None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
-
         with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
@@ -118,3 +89,124 @@ class RepoDependency(LocalDependency):
         # origin project url and rev_lock, and it makes RepoDependency
         # immutable, hence its impossible for checksum to change.
         return False
+
+    def get_used_objs(
+        self, **kwargs
+    ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
+        used, _ = self._get_used_and_obj(**kwargs)
+        return used
+
+    def _get_used_and_obj(
+        self, obj_only=False, **kwargs
+    ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]:
+        from dvc.config import NoRemoteError
+        from dvc.exceptions import NoOutputOrStageError, PathMissingError
+        from dvc.objects.stage import stage
+        from dvc.objects.tree import Tree
+
+        local_odb = self.repo.odb.local
+        locked = kwargs.pop("locked", True)
+        with self._make_repo(
+            locked=locked, cache_dir=local_odb.cache_dir
+        ) as repo:
+            used_obj_ids = defaultdict(set)
+            rev = repo.get_rev()
+            if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
+                self.def_repo[self.PARAM_REV_LOCK] = rev
+
+            path_info = PathInfo(repo.root_dir) / str(self.def_path)
+            if not obj_only:
+                try:
+                    for odb, obj_ids in repo.used_objs(
+                        [os.fspath(path_info)],
+                        force=True,
+                        jobs=kwargs.get("jobs"),
+                        recursive=True,
+                    ).items():
+                        if odb is None:
+                            odb = repo.cloud.get_remote_odb()
+                            odb.read_only = True
+                        self._check_circular_import(odb, obj_ids)
+                        used_obj_ids[odb].update(obj_ids)
+                except (NoRemoteError, NoOutputOrStageError):
+                    pass
+
+            try:
+                staging, staged_obj = stage(
+                    local_odb,
+                    path_info,
+                    repo.repo_fs,
+                    local_odb.fs.PARAM_CHECKSUM,
+                )
+            except FileNotFoundError as exc:
+                raise PathMissingError(
+                    self.def_path, self.def_repo[self.PARAM_URL]
+                ) from exc
+            staging = copy(staging)
+            staging.read_only = True
+
+            self._staged_objs[rev] = staged_obj
+            used_obj_ids[staging].add(staged_obj.hash_info)
+            if isinstance(staged_obj, Tree):
+                used_obj_ids[staging].update(
+                    entry.hash_info for _, entry in staged_obj
+                )
+            return used_obj_ids, staged_obj
+
+    def _check_circular_import(self, odb, obj_ids):
+        from dvc.exceptions import CircularImportError
+        from dvc.fs.repo import RepoFileSystem
+        from dvc.objects.db.reference import ReferenceObjectDB
+        from dvc.objects.tree import Tree
+
+        if not isinstance(odb, ReferenceObjectDB):
+            return
+
+        def iter_objs():
+            for hash_info in obj_ids:
+                if hash_info.isdir:
+                    tree = Tree.load(odb, hash_info)
+                    yield from (odb.get(entry.hash_info) for _, entry in tree)
+                else:
+                    yield odb.get(hash_info)
+
+        checked_urls = set()
+        for obj in iter_objs():
+            if not isinstance(obj.fs, RepoFileSystem):
+                continue
+            if (
+                obj.fs.repo_url in checked_urls
+                or obj.fs.root_dir in checked_urls
+            ):
+                continue
+            self_url = self.repo.url or self.repo.root_dir
+            if (
+                obj.fs.repo_url is not None
+                and obj.fs.repo_url == self_url
+                or obj.fs.root_dir == self.repo.root_dir
+            ):
+                raise CircularImportError(self, obj.fs.repo_url, self_url)
+            checked_urls.update([obj.fs.repo_url, obj.fs.root_dir])
+
+    def get_obj(self, filter_info=None, **kwargs):
+        locked = kwargs.get("locked", True)
+        rev = self._get_rev(locked=locked)
+        if rev in self._staged_objs:
+            return self._staged_objs[rev]
+        _, obj = self._get_used_and_obj(
+            obj_only=True, filter_info=filter_info, **kwargs
+        )
+        return obj
+
+    def _make_repo(self, locked=True, **kwargs):
+        from dvc.external_repo import external_repo
+
+        d = self.def_repo
+        rev = self._get_rev(locked=locked)
+        return external_repo(d[self.PARAM_URL], rev=rev, **kwargs)
+
+    def _get_rev(self, locked=True):
+        d = self.def_repo
+        return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(
+            self.PARAM_REV
+        )

dvc/dependency/s3.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.s3 import S3Output
-
-
-class S3Dependency(BaseDependency, S3Output):
-    pass

dvc/dependency/ssh.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.ssh import SSHOutput
-
-
-class SSHDependency(BaseDependency, SSHOutput):
-    pass

dvc/dependency/webdav.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.webdav import WebDAVFileSystem
-
-
-class WebDAVDependency(BaseDependency, BaseOutput):
-    FS_CLS = WebDAVFileSystem

dvc/dependency/webdavs.py

@@ -1,6 +0,0 @@
-from ..fs.webdavs import WebDAVSFileSystem
-from .webdav import WebDAVDependency
-
-
-class WebDAVSDependency(WebDAVDependency):
-    FS_CLS = WebDAVSFileSystem

dvc/dependency/webhdfs.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.webhdfs import WebHDFSOutput
-
-
-class WebHDFSDependency(BaseDependency, WebHDFSOutput):
-    pass

dvc/dvcfile.py

@@ -37,8 +37,12 @@ PIPELINE_LOCK = "dvc.lock"
 
 
 class FileIsGitIgnored(DvcException):
-    def __init__(self, path):
-        super().__init__(f"'{path}' is git-ignored.")
+    def __init__(self, path, pipeline_file=False):
+        super().__init__(
+            "{}'{}' is git-ignored.".format(
+                "bad DVC file name " if pipeline_file else "", path
+            )
+        )
 
 
 class LockfileCorruptedError(DvcException):
@@ -66,7 +70,19 @@ def is_lock_file(path):
     return os.path.basename(path) == PIPELINE_LOCK
 
 
-def check_dvc_filename(path):
+def is_git_ignored(repo, path):
+    from dvc.fs.local import LocalFileSystem
+    from dvc.scm.base import NoSCMError
+
+    try:
+        return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(
+            path
+        )
+    except NoSCMError:
+        return False
+
+
+def check_dvcfile_path(repo, path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
             "bad DVC file name '{}'. DVC files should be named "
@@ -75,6 +91,9 @@ def check_dvc_filename(path):
             )
         )
 
+    if is_git_ignored(repo, path):
+        raise FileIsGitIgnored(relpath(path), True)
+
 
 class FileMixin:
     SCHEMA = None
@@ -105,18 +124,15 @@ class FileMixin:
         return relpath(self.path)
 
     def exists(self):
-        return self.repo.fs.exists(self.path)
+        is_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+        return self.repo.fs.exists(self.path) and not is_ignored
 
     def _is_git_ignored(self):
-        from dvc.fs.local import LocalFileSystem
-
-        return isinstance(
-            self.repo.fs, LocalFileSystem
-        ) and self.repo.scm.is_ignored(self.path)
+        return is_git_ignored(self.repo, self.path)
 
     def _verify_filename(self):
         if self.verify:
-            check_dvc_filename(self.path)
+            check_dvcfile_path(self.repo, self.path)
 
     def _check_gitignored(self):
         if self._is_git_ignored():
@@ -129,8 +145,10 @@ class FileMixin:
         # 3. path doesn't represent a regular file
         # 4. when the file is git ignored
         if not self.exists():
-            is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
-            raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
+            dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+            raise StageFileDoesNotExistError(
+                self.path, dvc_ignored=dvc_ignored
+            )
 
         self._verify_filename()
         if not self.repo.fs.isfile(self.path):
@@ -181,10 +199,8 @@ class SingleStageFile(FileMixin):
 
         assert not isinstance(stage, PipelineStage)
         if self.verify:
-            check_dvc_filename(self.path)
-        logger.debug(
-            "Saving information to '{file}'.".format(file=relpath(self.path))
-        )
+            check_dvcfile_path(self.repo, self.path)
+        logger.debug(f"Saving information to '{relpath(self.path)}'.")
         dump_yaml(self.path, serialize.to_single_stage_file(stage))
         self.repo.scm.track_file(self.relpath)
 
@@ -218,7 +234,7 @@ class PipelineFile(FileMixin):
 
         assert isinstance(stage, PipelineStage)
         if self.verify:
-            check_dvc_filename(self.path)
+            check_dvcfile_path(self.repo, self.path)
 
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)
@@ -282,7 +298,7 @@ class PipelineFile(FileMixin):
         if not self.exists():
             return
 
-        with open(self.path, "r", encoding="utf-8") as f:
+        with open(self.path, encoding="utf-8") as f:
             d = parse_yaml_for_update(f.read(), self.path)
 
         self.validate(d, self.path)

dvc/env.py

@@ -6,3 +6,7 @@ DVCLIVE_PATH = "DVCLIVE_PATH"
 DVCLIVE_SUMMARY = "DVCLIVE_SUMMARY"
 DVCLIVE_HTML = "DVCLIVE_HTML"
 DVCLIVE_RESUME = "DVCLIVE_RESUME"
+DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY"
+DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE"
+DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH"
+DVC_NO_ANALYTICS = "DVC_NO_ANALYTICS"

dvc/exceptions.py

@@ -1,5 +1,4 @@
 """Exceptions raised by the dvc."""
-from typing import List
 
 
 class DvcException(Exception):
@@ -35,7 +34,7 @@ class OutputDuplicationError(DvcException):
             )
         else:
             msg = "output '{}' is already specified in stages:\n{}".format(
-                output, "\n".join(f"\t- {s.addressing}" for s in stages),
+                output, "\n".join(f"\t- {s.addressing}" for s in stages)
             )
         super().__init__(msg)
         self.stages = stages
@@ -173,36 +172,6 @@ class BadMetricError(DvcException):
         )
 
 
-class MetricsError(DvcException):
-    pass
-
-
-class NoMetricsParsedError(MetricsError):
-    def __init__(self, command):
-        super().__init__(
-            f"Could not parse {command} files. Use `-v` option to see more "
-            "details."
-        )
-
-
-class NoMetricsFoundError(MetricsError):
-    def __init__(self, command, run_options):
-        super().__init__(
-            f"No {command} files in this repository. "
-            f"Use `{run_options}` options for "
-            f"`dvc run` to mark stage outputs as {command}."
-        )
-
-
-class MetricDoesNotExistError(MetricsError):
-    def __init__(self, targets: List[str]):
-        if len(targets) == 1:
-            msg = "'{}' does not exist."
-        else:
-            msg = "'{}' do not exist."
-        super().__init__(msg.format(", ".join(targets)))
-
-
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super().__init__(
@@ -259,18 +228,21 @@ class GitHookAlreadyExistsError(DvcException):
         )
 
 
-class DownloadError(DvcException):
+class FileTransferError(DvcException):
+    _METHOD = "transfer"
+
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(f"{amount} files failed to download")
+        super().__init__(f"{amount} files failed to {self._METHOD}")
 
 
-class UploadError(DvcException):
-    def __init__(self, amount):
-        self.amount = amount
+class DownloadError(FileTransferError):
+    _METHOD = "download"
 
-        super().__init__(f"{amount} files failed to upload")
+
+class UploadError(FileTransferError):
+    _METHOD = "upload"
 
 
 class CheckoutError(DvcException):
@@ -283,7 +255,7 @@ class CheckoutError(DvcException):
         m = (
             "Checkout failed for following targets:\n{}\nIs your "
             "cache up to date?\n{}".format(
-                "\n".join(targets), error_link("missing-files"),
+                "\n".join(targets), error_link("missing-files")
             )
         )
         super().__init__(m)
@@ -380,7 +352,15 @@ class CacheLinkError(DvcException):
 
     def __init__(self, path_infos):
         msg = "No possible cache link types for '{}'. {}".format(
-            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK,
+            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK
         )
         super().__init__(msg)
         self.path_infos = path_infos
+
+
+class CircularImportError(DvcException):
+    def __init__(self, dep, a, b):
+        super().__init__(
+            f"'{dep}' contains invalid circular import. "
+            f"DVC repo '{a}' already imports from '{b}'."
+        )

dvc/external_repo.py

@@ -36,23 +36,12 @@ def external_repo(
     rev = rev or "refs/remotes/origin/HEAD"
 
     cache_config = {
-        "cache": {
-            "dir": cache_dir or _get_cache_dir(url),
-            "type": cache_types,
-        }
+        "cache": {"dir": cache_dir or _get_cache_dir(url), "type": cache_types}
     }
 
     config = _get_remote_config(url) if os.path.isdir(url) else {}
     config.update(cache_config)
 
-    def make_repo(path, **_kwargs):
-        _config = cache_config.copy()
-        if os.path.isdir(url):
-            rel = os.path.relpath(path, _kwargs["scm"].root_dir)
-            repo_path = os.path.join(url, rel)
-            _config.update(_get_remote_config(repo_path))
-        return Repo(path, config=_config, **_kwargs)
-
     root_dir = path if for_write else os.path.realpath(path)
     repo_kwargs = dict(
         root_dir=root_dir,
@@ -60,7 +49,7 @@ def external_repo(
         scm=None if for_write else Git(root_dir),
         rev=None if for_write else rev,
         config=config,
-        repo_factory=make_repo,
+        repo_factory=erepo_factory(url, cache_config),
         **kwargs,
     )
 
@@ -90,6 +79,18 @@ def external_repo(
             _remove(path)
 
 
+def erepo_factory(url, cache_config, *args, **kwargs):
+    def make_repo(path, **_kwargs):
+        _config = cache_config.copy()
+        if os.path.isdir(url):
+            rel = os.path.relpath(path, _kwargs["scm"].root_dir)
+            repo_path = os.path.join(url, rel)
+            _config.update(_get_remote_config(repo_path))
+        return Repo(path, config=_config, **_kwargs)
+
+    return make_repo
+
+
 CLONES: Dict[str, str] = {}
 CACHE_DIRS: Dict[str, str] = {}
 

dvc/fs/__init__.py

@@ -2,6 +2,7 @@ import posixpath
 from urllib.parse import urlparse
 
 from ..scheme import Schemes
+from . import utils  # noqa: F401
 from .azure import AzureFileSystem
 from .gdrive import GDriveFileSystem
 from .gs import GSFileSystem
@@ -12,8 +13,7 @@ from .local import LocalFileSystem
 from .oss import OSSFileSystem
 from .s3 import S3FileSystem
 from .ssh import SSHFileSystem
-from .webdav import WebDAVFileSystem
-from .webdavs import WebDAVSFileSystem
+from .webdav import WebDAVFileSystem, WebDAVSFileSystem
 from .webhdfs import WebHDFSFileSystem
 
 FS_MAP = {
@@ -33,15 +33,21 @@ FS_MAP = {
 }
 
 
-def get_fs_cls(remote_conf):
-    scheme = urlparse(remote_conf["url"]).scheme
+def get_fs_cls(remote_conf, scheme=None):
+    if not scheme:
+        scheme = urlparse(remote_conf["url"]).scheme
     return FS_MAP.get(scheme, LocalFileSystem)
 
 
 def get_fs_config(config, **kwargs):
     name = kwargs.get("name")
     if name:
-        remote_conf = config["remote"][name.lower()]
+        try:
+            remote_conf = config["remote"][name.lower()]
+        except KeyError:
+            from dvc.config import RemoteNotFoundError
+
+            raise RemoteNotFoundError(f"remote '{name}' doesn't exist")
     else:
         remote_conf = kwargs
     return _resolve_remote_refs(config, remote_conf)
@@ -82,9 +88,35 @@ def get_cloud_fs(repo, **kwargs):
     from dvc.config import ConfigError
     from dvc.config_schema import SCHEMA, Invalid
 
-    remote_conf = get_fs_config(repo.config, **kwargs)
+    repo_config = repo.config if repo else {}
+    core_config = repo_config.get("core", {})
+
+    remote_conf = get_fs_config(repo_config, **kwargs)
     try:
         remote_conf = SCHEMA["remote"][str](remote_conf)
     except Invalid as exc:
         raise ConfigError(str(exc)) from None
-    return get_fs_cls(remote_conf)(repo, remote_conf)
+
+    if "jobs" not in remote_conf:
+        jobs = core_config.get("jobs")
+        if jobs:
+            remote_conf["jobs"] = jobs
+
+    if "checksum_jobs" not in remote_conf:
+        checksum_jobs = core_config.get("checksum_jobs")
+        if checksum_jobs:
+            remote_conf["checksum_jobs"] = checksum_jobs
+
+    cls = get_fs_cls(remote_conf)
+
+    if cls == GDriveFileSystem and repo:
+        remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
+
+    url = remote_conf.pop("url")
+    path_info = cls.PATH_CLS(
+        cls._strip_protocol(url)  # pylint:disable=protected-access
+    )
+
+    extras = cls._get_kwargs_from_urls(url)  # pylint:disable=protected-access
+    conf = {**extras, **remote_conf}  # remote config takes priority
+    return cls, conf, path_info

dvc/fs/_metadata.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, List
 
-from dvc.output import BaseOutput
+from dvc.output import Output
 from dvc.path_info import PathInfo
 
 if TYPE_CHECKING:
@@ -33,7 +33,7 @@ class Metadata:
     # optional fields
     isdir: bool = False  # is it a directory?
     is_exec: bool = False  # is it an executable?
-    outs: List[BaseOutput] = field(default_factory=list)  # list of outputs
+    outs: List[Output] = field(default_factory=list)  # list of outputs
 
     def __post_init__(self):
         self.output_exists = bool(self.outs)

dvc/fs/azure.py

@@ -2,14 +2,16 @@ import logging
 import os
 import threading
 
-from funcy import cached_property, wrap_prop
+from fsspec.asyn import fsspec_loop
+from fsspec.utils import infer_storage_options
+from funcy import cached_property, memoize, wrap_prop
 
 from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 from dvc.utils import format_link
 
-from .fsspec_wrapper import FSSpecWrapper
+from .fsspec_wrapper import ObjectFSWrapper
 
 logger = logging.getLogger(__name__)
 _DEFAULT_CREDS_STEPS = (
@@ -23,7 +25,21 @@ class AzureAuthError(DvcException):
     pass
 
 
-class AzureFileSystem(FSSpecWrapper):
+@memoize
+def _az_config():
+    # NOTE: ideally we would've used get_default_cli().config from
+    # azure.cli.core, but azure-cli-core has a lot of conflicts with other
+    # dependencies. So instead we are just use knack directly
+    from knack.config import CLIConfig
+
+    config_dir = os.getenv(
+        "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
+    )
+    return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
+
+
+# pylint:disable=abstract-method
+class AzureFileSystem(ObjectFSWrapper):
     scheme = Schemes.AZURE
     PATH_CLS = CloudURLInfo
     PARAM_CHECKSUM = "etag"
@@ -34,21 +50,23 @@ class AzureFileSystem(FSSpecWrapper):
         "azure-identity": "azure.identity",
     }
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    @classmethod
+    def _strip_protocol(cls, path: str):
+        bucket = infer_storage_options(path).get("host")
+        if bucket:
+            return path
 
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
+        bucket = _az_config().get("storage", "container_name", None)
+        return f"azure://{bucket}"
 
-        if not self.bucket:
-            container = self._az_config.get("storage", "container_name", None)
-            url = f"azure://{container}"
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        ops = infer_storage_options(urlpath)
+        if "host" in ops:
+            return {"bucket": ops["host"]}
+        return {}
 
-        self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
-
-    def _prepare_credentials(self, config):
+    def _prepare_credentials(self, **config):
         from azure.identity.aio import DefaultAzureCredential
 
         # Disable spam from failed cred types for DefaultAzureCredential
@@ -57,28 +75,54 @@ class AzureFileSystem(FSSpecWrapper):
         login_info = {}
         login_info["connection_string"] = config.get(
             "connection_string",
-            self._az_config.get("storage", "connection_string", None),
+            _az_config().get("storage", "connection_string", None),
         )
         login_info["account_name"] = config.get(
-            "account_name", self._az_config.get("storage", "account", None)
+            "account_name", _az_config().get("storage", "account", None)
         )
         login_info["account_key"] = config.get(
-            "account_key", self._az_config.get("storage", "key", None)
+            "account_key", _az_config().get("storage", "key", None)
         )
         login_info["sas_token"] = config.get(
-            "sas_token", self._az_config.get("storage", "sas_token", None)
+            "sas_token", _az_config().get("storage", "sas_token", None)
         )
         login_info["tenant_id"] = config.get("tenant_id")
         login_info["client_id"] = config.get("client_id")
         login_info["client_secret"] = config.get("client_secret")
 
-        if not any(
-            value for key, value in login_info.items() if key != "account_name"
-        ):
-            login_info["credential"] = DefaultAzureCredential(
-                exclude_interactive_browser_credential=False
+        if not (login_info["account_name"] or login_info["connection_string"]):
+            raise AzureAuthError(
+                "Authentication to Azure Blob Storage requires either "
+                "account_name or connection_string.\nLearn more about "
+                "configuration settings at "
+                + format_link("https://man.dvc.org/remote/modify")
             )
 
+        any_secondary = any(
+            value for key, value in login_info.items() if key != "account_name"
+        )
+        if (
+            login_info["account_name"]
+            and not any_secondary
+            and not config.get("allow_anonymous_login", False)
+        ):
+            with fsspec_loop():
+                login_info["credential"] = DefaultAzureCredential(
+                    exclude_interactive_browser_credential=False,
+                    exclude_environment_credential=config.get(
+                        "exclude_environment_credential", False
+                    ),
+                    exclude_visual_studio_code_credential=config.get(
+                        "exclude_visual_studio_code_credential", False
+                    ),
+                    exclude_shared_token_cache_credential=config.get(
+                        "exclude_shared_token_cache_credential", False
+                    ),
+                    exclude_managed_identity_credential=config.get(
+                        "exclude_managed_identity_credential", False
+                    ),
+                )
+
         for login_method, required_keys in [  # noqa
             ("connection string", ["connection_string"]),
             (
@@ -101,18 +145,6 @@ class AzureFileSystem(FSSpecWrapper):
         self.login_method = login_method
         return login_info
 
-    @cached_property
-    def _az_config(self):
-        # NOTE: ideally we would've used get_default_cli().config from
-        # azure.cli.core, but azure-cli-core has a lot of conflicts with other
-        # dependencies. So instead we are just use knack directly
-        from knack.config import CLIConfig
-
-        config_dir = os.getenv(
-            "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
-        )
-        return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
-
     @wrap_prop(threading.Lock())
     @cached_property
     def fs(self):
@@ -120,16 +152,10 @@ class AzureFileSystem(FSSpecWrapper):
         from azure.core.exceptions import AzureError
 
         try:
-            file_system = AzureBlobFileSystem(**self.fs_args)
-            if self.bucket not in [
-                container.rstrip("/") for container in file_system.ls("/")
-            ]:
-                file_system.mkdir(self.bucket)
+            return AzureBlobFileSystem(**self.fs_args)
         except (ValueError, AzureError) as e:
             raise AzureAuthError(
                 f"Authentication to Azure Blob Storage via {self.login_method}"
                 " failed.\nLearn more about configuration settings at"
                 f" {format_link('https://man.dvc.org/remote/modify')}"
             ) from e
-
-        return file_system

dvc/fs/base.py

@@ -4,14 +4,11 @@ from functools import partial, partialmethod
 from multiprocessing import cpu_count
 from typing import Any, ClassVar, Dict, FrozenSet, Optional
 
-from funcy import cached_property
-
 from dvc.exceptions import DvcException
 from dvc.path_info import URLInfo
 from dvc.progress import Tqdm
 from dvc.utils import tmp_fname
 from dvc.utils.fs import makedirs, move
-from dvc.utils.http import open_url
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +32,8 @@ class RemoteMissingDepsError(DvcException):
 
 
 class BaseFileSystem:
+    sep = "/"
+
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
     PATH_CLS = URLInfo  # type: Any
@@ -54,29 +53,24 @@ class BaseFileSystem:
     PARAM_CHECKSUM: ClassVar[Optional[str]] = None
     DETAIL_FIELDS: FrozenSet[str] = frozenset()
 
-    def __init__(self, repo, config):
-        self.repo = repo
-        self.config = config
+    def __init__(self, **kwargs):
+        self._check_requires(**kwargs)
 
-        self._check_requires()
+        self.jobs = kwargs.get("jobs") or self._JOBS
+        self.hash_jobs = kwargs.get("checksum_jobs") or self.HASH_JOBS
+        self._config = kwargs
 
-        self.path_info = None
+    @property
+    def config(self):
+        return self._config
 
-    @cached_property
-    def jobs(self):
-        return (
-            self.config.get("jobs")
-            or (self.repo and self.repo.config["core"].get("jobs"))
-            or self._JOBS
-        )
+    @classmethod
+    def _strip_protocol(cls, path: str):
+        return path
 
-    @cached_property
-    def hash_jobs(self):
-        return (
-            self.config.get("checksum_jobs")
-            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
-            or self.HASH_JOBS
-        )
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):  # pylint:disable=unused-argument
+        return {}
 
     @classmethod
     def get_missing_deps(cls):
@@ -91,7 +85,7 @@ class BaseFileSystem:
 
         return missing
 
-    def _check_requires(self):
+    def _check_requires(self, **kwargs):
         from ..scheme import Schemes
         from ..utils import format_link
         from ..utils.pkg import PKG
@@ -100,7 +94,7 @@ class BaseFileSystem:
         if not missing:
             return
 
-        url = self.config.get("url", f"{self.scheme}://")
+        url = kwargs.get("url", f"{self.scheme}://")
 
         scheme = self.scheme
         if scheme == Schemes.WEBDAVS:
@@ -130,16 +124,13 @@ class BaseFileSystem:
             f"dependencies: {missing}. {hint}"
         )
 
-    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
-        if hasattr(self, "_generate_download_url"):
-            # pylint:disable=no-member
-            func = self._generate_download_url  # type: ignore[attr-defined]
-            get_url = partial(func, path_info)
-            return open_url(get_url, mode=mode, encoding=encoding, **kwargs)
+    def checksum(self, path_info) -> str:
+        raise NotImplementedError
 
+    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
         raise RemoteActionNotImplemented("open", self.scheme)
 
-    def exists(self, path_info, use_dvcignore=True) -> bool:
+    def exists(self, path_info) -> bool:
         raise NotImplementedError
 
     # pylint: disable=unused-argument
@@ -166,6 +157,10 @@ class BaseFileSystem:
         """Check if this file is an independent copy."""
         return False  # We can't be sure by default
 
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        """Return a generator with (root, dirs, files)."""
+        raise NotImplementedError
+
     def walk_files(self, path_info, **kwargs):
         """Return a generator with `PathInfo`s to all the files.
 
@@ -178,7 +173,7 @@ class BaseFileSystem:
     def ls(self, path_info, detail=False):
         raise RemoteActionNotImplemented("ls", self.scheme)
 
-    def find(self, path_info, detail=False):
+    def find(self, path_info, detail=False, prefix=None):
         raise RemoteActionNotImplemented("find", self.scheme)
 
     def is_empty(self, path_info):
@@ -193,7 +188,7 @@ class BaseFileSystem:
     def remove(self, path_info):
         raise RemoteActionNotImplemented("remove", self.scheme)
 
-    def makedirs(self, path_info):
+    def makedirs(self, path_info, **kwargs):
         """Optional: Implement only if the remote needs to create
         directories before copying/linking/moving data
         """
@@ -222,9 +217,7 @@ class BaseFileSystem:
             return False
         return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
-    def upload(
-        self, from_info, to_info, name=None, no_progress_bar=False,
-    ):
+    def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):
             raise RemoteActionNotImplemented("upload", self.scheme)
 
@@ -245,14 +238,23 @@ class BaseFileSystem:
             no_progress_bar=no_progress_bar,
         )
 
-    def upload_fobj(self, fobj, to_info, no_progress_bar=False, **pbar_args):
+    def upload_fobj(
+        self, fobj, to_info, no_progress_bar=False, size=None, **pbar_args
+    ):
         if not hasattr(self, "_upload_fobj"):
             raise RemoteActionNotImplemented("upload_fobj", self.scheme)
 
         with Tqdm.wrapattr(
-            fobj, "read", disable=no_progress_bar, bytes=True, **pbar_args
+            fobj,
+            "read",
+            disable=no_progress_bar,
+            bytes=True,
+            total=size,
+            **pbar_args,
         ) as wrapped:
-            self._upload_fobj(wrapped, to_info)  # pylint: disable=no-member
+            self._upload_fobj(  # pylint: disable=no-member
+                wrapped, to_info, size=size
+            )
 
     def download(
         self,
@@ -279,14 +281,14 @@ class BaseFileSystem:
 
         if not _only_file and self.isdir(from_info):
             return self._download_dir(
-                from_info, to_info, name, no_progress_bar, jobs, **kwargs,
+                from_info, to_info, name, no_progress_bar, jobs, **kwargs
             )
-        return self._download_file(from_info, to_info, name, no_progress_bar,)
+        return self._download_file(from_info, to_info, name, no_progress_bar)
 
     download_file = partialmethod(download, _only_file=True)
 
     def _download_dir(
-        self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
+        self, from_info, to_info, name, no_progress_bar, jobs, **kwargs
     ):
         from_infos = list(self.walk_files(from_info, **kwargs))
         if not from_infos:
@@ -303,7 +305,7 @@ class BaseFileSystem:
             disable=no_progress_bar,
         ) as pbar:
             download_files = pbar.wrap_fn(
-                partial(self._download_file, name=name, no_progress_bar=True,)
+                partial(self._download_file, name=name, no_progress_bar=True)
             )
             max_workers = jobs or self.jobs
             with ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -327,9 +329,7 @@ class BaseFileSystem:
                             entry.cancel()
                         raise exc
 
-    def _download_file(
-        self, from_info, to_info, name, no_progress_bar,
-    ):
+    def _download_file(self, from_info, to_info, name, no_progress_bar):
         makedirs(to_info.parent, exist_ok=True)
 
         logger.debug("Downloading '%s' to '%s'", from_info, to_info)

dvc/fs/dvc.py

@@ -10,7 +10,7 @@ from ._metadata import Metadata
 from .base import BaseFileSystem
 
 if typing.TYPE_CHECKING:
-    from dvc.output.base import BaseOutput
+    from dvc.output import Output
 
 
 logger = logging.getLogger(__name__)
@@ -23,11 +23,18 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         repo: DVC repo.
     """
 
+    sep = os.sep
+
     scheme = "local"
     PARAM_CHECKSUM = "md5"
 
-    def __init__(self, repo):
-        super().__init__(repo, {"url": repo.root_dir})
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.repo = kwargs["repo"]
+
+    @property
+    def config(self):
+        raise NotImplementedError
 
     def _find_outs(self, path, *args, **kwargs):
         outs = self.repo.find_outs_by_path(path, *args, **kwargs)
@@ -42,20 +49,20 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         return outs
 
     def _get_granular_hash(
-        self, path_info: PathInfo, out: "BaseOutput", remote=None
+        self, path_info: PathInfo, out: "Output", remote=None
     ):
         assert isinstance(path_info, PathInfo)
         # NOTE: use string paths here for performance reasons
         key = tuple(relpath(path_info, out.path_info).split(os.sep))
         out.get_dir_cache(remote=remote)
+        if out.obj is None:
+            raise FileNotFoundError
         obj = out.obj.trie.get(key)
         if obj:
             return obj.hash_info
         raise FileNotFoundError
 
-    def open(  # type: ignore
-        self, path: PathInfo, mode="r", encoding=None, remote=None, **kwargs
-    ):  # pylint: disable=arguments-differ
+    def _get_fs_path(self, path: PathInfo, remote=None):
         try:
             outs = self._find_outs(path, strict=False)
         except OutputNotFoundError as exc:
@@ -67,37 +74,45 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             raise IsADirectoryError
 
         out = outs[0]
+
+        if not out.hash_info:
+            raise FileNotFoundError
+
         if out.changed_cache(filter_info=path):
             from dvc.config import NoRemoteError
 
             try:
-                remote_obj = self.repo.cloud.get_remote(remote)
-            except NoRemoteError:
-                raise FileNotFoundError
+                remote_odb = self.repo.cloud.get_remote_odb(remote)
+            except NoRemoteError as exc:
+                raise FileNotFoundError from exc
             if out.is_dir_checksum:
                 checksum = self._get_granular_hash(path, out).value
             else:
                 checksum = out.hash_info.value
-            remote_info = remote_obj.odb.hash_to_path_info(checksum)
-            return remote_obj.fs.open(
-                remote_info, mode=mode, encoding=encoding
-            )
+            remote_info = remote_odb.hash_to_path_info(checksum)
+            return remote_odb.fs, remote_info
 
         if out.is_dir_checksum:
             checksum = self._get_granular_hash(path, out).value
             cache_path = out.odb.hash_to_path_info(checksum).url
         else:
             cache_path = out.cache_path
-        return open(cache_path, mode=mode, encoding=encoding)
+        return out.odb.fs, cache_path
 
-    def exists(self, path):  # pylint: disable=arguments-differ
+    def open(  # type: ignore
+        self, path: PathInfo, mode="r", encoding=None, **kwargs
+    ):  # pylint: disable=arguments-renamed
+        fs, fspath = self._get_fs_path(path, **kwargs)
+        return fs.open(fspath, mode=mode, encoding=encoding)
+
+    def exists(self, path):  # pylint: disable=arguments-renamed
         try:
             self.metadata(path)
             return True
         except FileNotFoundError:
             return False
 
-    def isdir(self, path):  # pylint: disable=arguments-differ
+    def isdir(self, path):  # pylint: disable=arguments-renamed
         try:
             meta = self.metadata(path)
             return meta.isdir
@@ -120,7 +135,7 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         except FileNotFoundError:
             return True
 
-    def isfile(self, path):  # pylint: disable=arguments-differ
+    def isfile(self, path):  # pylint: disable=arguments-renamed
         try:
             meta = self.metadata(path)
             return meta.isfile
@@ -226,7 +241,7 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
     def info(self, path_info):
         meta = self.metadata(path_info)
-        ret = {"type": "dir" if meta.isdir else "file"}
+        ret = {"type": "directory" if meta.isdir else "file"}
         if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
             hash_info = meta.outs[0].hash_info
             ret["size"] = hash_info.size
@@ -240,3 +255,16 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 ret[obj.hash_info.name] = obj.hash_info.value
 
         return ret
+
+    def _download(self, from_info, to_file, **kwargs):
+        fs, path = self._get_fs_path(from_info)
+        fs._download(  # pylint: disable=protected-access
+            path, to_file, **kwargs
+        )
+
+    def checksum(self, path_info):
+        info = self.info(path_info)
+        md5 = info.get("md5")
+        if md5:
+            return md5
+        raise NotImplementedError

dvc/fs/fsspec_wrapper.py

@@ -2,36 +2,41 @@ import os
 import shutil
 from functools import lru_cache
 
+from funcy import cached_property
+
 from dvc.progress import Tqdm
 
 from .base import BaseFileSystem
+from .local import LocalFileSystem
 
 
 # pylint: disable=no-member
 class FSSpecWrapper(BaseFileSystem):
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-        self.fs_args = {"skip_instance_cache": True}
-        self.fs_args.update(self._prepare_credentials(config))
+    TRAVERSE_PREFIX_LEN = 2
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.fs_args = {"skip_instance_cache": True}
+        self.fs_args.update(self._prepare_credentials(**kwargs))
+
+    @cached_property
     def fs(self):
         raise NotImplementedError
 
-    @lru_cache(512)
     def _with_bucket(self, path):
-        if isinstance(path, self.PATH_CLS):
-            return f"{path.bucket}/{path.path}"
-        return path
+        return str(path)
 
     def _strip_bucket(self, entry):
-        try:
-            bucket, path = entry.split("/", 1)
-        except ValueError:
-            # If there is no path attached, only returns
-            # the bucket (top-level).
-            bucket, path = entry, None
+        return entry
 
-        return path or bucket
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        from fsspec.utils import infer_storage_options
+
+        options = infer_storage_options(urlpath)
+        options.pop("path", None)
+        options.pop("protocol", None)
+        return options
 
     def _strip_buckets(self, entries, detail=False):
         for entry in entries:
@@ -43,37 +48,32 @@ class FSSpecWrapper(BaseFileSystem):
             yield entry
 
     def _entry_hook(self, entry):
-        """Simple hook method to be overriden when wanted to process
+        """Simple hook method to be overridden when wanted to process
         entries within info() and ls(detail=True) calls"""
         return entry
 
-    def _prepare_credentials(self, config):  # pylint: disable=unused-argument
+    def _prepare_credentials(
+        self, **config
+    ):  # pylint: disable=unused-argument
         """Prepare the arguments for authentication to the
         host filesystem"""
         return {}
 
-    def isdir(self, path_info):
-        if not self.exists(path_info):
-            return False
+    def _isdir(self, path_info):
+        return self.fs.isdir(self._with_bucket(path_info))
 
-        # Directory in object storages are interpreted differently
-        # among different fsspec providers, so this logic is a temporary
-        # measure for us to adapt as of now. It checks whether it is a
-        # directory (as in a prefix with contents) or whether it is an empty
-        # file where it's name ends with a forward slash
-        entry = self.info(path_info)
-        return entry["type"] == "directory" or (
-            entry["size"] == 0
-            and entry["type"] == "file"
-            and entry["name"].endswith("/")
-        )
+    def isdir(self, path_info):
+        try:
+            return self._isdir(path_info)
+        except FileNotFoundError:
+            return False
 
     def isfile(self, path_info):
-        if not self.exists(path_info):
+        try:
+            return not self._isdir(path_info)
+        except FileNotFoundError:
             return False
 
-        return not self.isdir(path_info)
-
     def is_empty(self, path_info):
         entry = self.info(path_info)
         if entry["type"] == "directory":
@@ -85,10 +85,14 @@ class FSSpecWrapper(BaseFileSystem):
     ):  # pylint: disable=arguments-differ
         return self.fs.open(self._with_bucket(path_info), mode=mode)
 
+    def checksum(self, path_info):
+        return self.fs.checksum(self._with_bucket(path_info))
+
     def copy(self, from_info, to_info):
+        self.makedirs(to_info.parent)
         self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
 
-    def exists(self, path_info, use_dvcignore=False):
+    def exists(self, path_info) -> bool:
         return self.fs.exists(self._with_bucket(path_info))
 
     def ls(self, path_info, detail=False):
@@ -96,28 +100,24 @@ class FSSpecWrapper(BaseFileSystem):
         files = self.fs.ls(path, detail=detail)
         yield from self._strip_buckets(files, detail=detail)
 
-    def find(self, path_info, detail=False):
+    # pylint: disable=unused-argument
+    def find(self, path_info, detail=False, prefix=None):
         path = self._with_bucket(path_info)
         files = self.fs.find(path, detail=detail)
         if detail:
             files = files.values()
 
-        # When calling find() on a file, it returns the same file in a list.
-        # For object-based storages, the same behavior applies to empty
-        # directories since they are represented as files. This condition
-        # checks whether we should yield an empty list (if it is an empty
-        # directory) or just yield the file itself.
-        if len(files) == 1 and files[0] == path and self.isdir(path_info):
-            return None
-
         yield from self._strip_buckets(files, detail=detail)
 
     def walk_files(self, path_info, **kwargs):
-        for file in self.find(path_info):
+        for file in self.find(path_info, **kwargs):
             yield path_info.replace(path=file)
 
+    def move(self, from_info, to_info):
+        self.fs.move(self._with_bucket(from_info), self._with_bucket(to_info))
+
     def remove(self, path_info):
-        self.fs.rm(self._with_bucket(path_info))
+        self.fs.rm_file(self._with_bucket(path_info))
 
     def info(self, path_info):
         info = self.fs.info(self._with_bucket(path_info)).copy()
@@ -125,20 +125,27 @@ class FSSpecWrapper(BaseFileSystem):
         info["name"] = self._strip_bucket(info["name"])
         return info
 
-    def _upload_fobj(self, fobj, to_info):
+    def makedirs(self, path_info, **kwargs):
+        self.fs.makedirs(
+            self._with_bucket(path_info), exist_ok=kwargs.pop("exist_ok", True)
+        )
+
+    def _upload_fobj(self, fobj, to_info, size=None):
+        self.makedirs(to_info.parent)
         with self.open(to_info, "wb") as fdest:
             shutil.copyfileobj(fobj, fdest, length=fdest.blocksize)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **kwargs
     ):
-        total = os.path.getsize(from_file)
+        self.makedirs(to_info.parent)
+        size = os.path.getsize(from_file)
         with open(from_file, "rb") as fobj:
             self.upload_fobj(
                 fobj,
-                self._with_bucket(to_info),
+                to_info,
+                size=size,
                 desc=name,
-                total=total,
                 no_progress_bar=no_progress_bar,
             )
         self.fs.invalidate_cache(self._with_bucket(to_info.parent))
@@ -159,3 +166,130 @@ class FSSpecWrapper(BaseFileSystem):
             ) as wrapped:
                 with open(to_file, "wb") as fdest:
                     shutil.copyfileobj(wrapped, fdest, length=fobj.blocksize)
+
+
+# pylint: disable=abstract-method
+class ObjectFSWrapper(FSSpecWrapper):
+    TRAVERSE_PREFIX_LEN = 3
+
+    @lru_cache(512)
+    def _with_bucket(self, path):
+        if isinstance(path, self.PATH_CLS):
+            return f"{path.bucket}/{path.path}"
+        return path
+
+    def _strip_bucket(self, entry):
+        try:
+            bucket, path = entry.split("/", 1)
+        except ValueError:
+            # If there is no path attached, only returns
+            # the bucket (top-level).
+            bucket, path = entry, None
+        return path or bucket
+
+    def makedirs(self, path_info, **kwargs):
+        # For object storages make this method a no-op. The original
+        # fs.makedirs() method will only check if the bucket exists
+        # and create if it doesn't though we don't want to support
+        # that behavior, and the check will cost some time so we'll
+        # simply ignore all mkdir()/makedirs() calls.
+        return None
+
+    def _isdir(self, path_info):
+        # Directory in object storages are interpreted differently
+        # among different fsspec providers, so this logic is a temporary
+        # measure for us to adapt as of now. It checks whether it is a
+        # directory (as in a prefix with contents) or whether it is an empty
+        # file where it's name ends with a forward slash
+
+        entry = self.info(path_info)
+        return entry["type"] == "directory" or (
+            entry["size"] == 0
+            and entry["type"] == "file"
+            and entry["name"].endswith("/")
+        )
+
+    def find(self, path_info, detail=False, prefix=None):
+        if prefix:
+            path = self._with_bucket(path_info.parent)
+            files = self.fs.find(
+                path, detail=detail, prefix=path_info.parts[-1]
+            )
+        else:
+            path = self._with_bucket(path_info)
+            files = self.fs.find(path, detail=detail)
+
+        if detail:
+            files = list(files.values())
+
+        # When calling find() on a file, it returns the same file in a list.
+        # For object-based storages, the same behavior applies to empty
+        # directories since they are represented as files. This condition
+        # checks whether we should yield an empty list (if it is an empty
+        # directory) or just yield the file itself.
+        if len(files) == 1 and files[0] == path and self.isdir(path_info):
+            return None
+
+        yield from self._strip_buckets(files, detail=detail)
+
+
+# pylint: disable=arguments-differ
+class NoDirectoriesMixin:
+    def isdir(self, *args, **kwargs):
+        return False
+
+    def isfile(self, *args, **kwargs):
+        return True
+
+    def find(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def walk(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def walk_files(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def ls(self, *args, **kwargs):
+        raise NotImplementedError
+
+
+_LOCAL_FS = LocalFileSystem()
+
+
+class CallbackMixin:
+    """Use the native ``get_file()``/``put_file()`` APIs
+    if the target filesystem supports callbacks."""
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **pbar_args
+    ):
+        with Tqdm(
+            desc=name,
+            disable=no_progress_bar,
+            bytes=True,
+            total=-1,
+            **pbar_args,
+        ) as pbar:
+            self.fs.put_file(
+                os.fspath(from_file),
+                self._with_bucket(to_info),
+                callback=pbar.as_callback(_LOCAL_FS, from_file),
+            )
+        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
+
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
+    ):
+        with Tqdm(
+            desc=name,
+            disable=no_progress_bar,
+            bytes=True,
+            total=-1,
+            **pbar_args,
+        ) as pbar:
+            self.fs.get_file(
+                self._with_bucket(from_info),
+                os.fspath(to_file),
+                callback=pbar.as_callback(self, from_info),
+            )

dvc/fs/gdrive.py

@@ -1,24 +1,18 @@
-import io
 import logging
 import os
 import posixpath
 import re
 import threading
-from collections import defaultdict
-from contextlib import contextmanager
 from urllib.parse import urlparse
 
-from funcy import cached_property, retry, wrap_prop, wrap_with
-from funcy.py3 import cat
+from funcy import cached_property, retry, wrap_prop
 
-from dvc.exceptions import DvcException, FileMissingError
+from dvc.exceptions import DvcException
 from dvc.path_info import CloudURLInfo
-from dvc.progress import Tqdm
 from dvc.scheme import Schemes
 from dvc.utils import format_link, tmp_fname
-from dvc.utils.stream import IterStream
 
-from .base import BaseFileSystem
+from .fsspec_wrapper import CallbackMixin, FSSpecWrapper
 
 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -85,7 +79,9 @@ class GDriveURLInfo(CloudURLInfo):
         self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
 
 
-class GDriveFileSystem(BaseFileSystem):
+class GDriveFileSystem(
+    CallbackMixin, FSSpecWrapper
+):  # pylint:disable=abstract-method
     scheme = Schemes.GDRIVE
     PATH_CLS = GDriveURLInfo
     PARAM_CHECKSUM = "checksum"
@@ -100,8 +96,8 @@ class GDriveFileSystem(BaseFileSystem):
     DEFAULT_GDRIVE_CLIENT_ID = "710796635688-iivsgbgsb6uv1fap6635dhvuei09o66c.apps.googleusercontent.com"  # noqa: E501
     DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
         self.path_info = self.PATH_CLS(config["url"])
 
@@ -126,20 +122,26 @@ class GDriveFileSystem(BaseFileSystem):
         self._client_id = config.get("gdrive_client_id")
         self._client_secret = config.get("gdrive_client_secret")
         self._validate_config()
+
+        tmp_dir = config["gdrive_credentials_tmp_dir"]
+        assert tmp_dir
+
         self._gdrive_service_credentials_path = tmp_fname(
-            os.path.join(self.repo.tmp_dir, "")
+            os.path.join(tmp_dir, "")
         )
         self._gdrive_user_credentials_path = (
-            tmp_fname(os.path.join(self.repo.tmp_dir, ""))
+            tmp_fname(os.path.join(tmp_dir, ""))
             if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
             else config.get(
                 "gdrive_user_credentials_file",
-                os.path.join(
-                    self.repo.tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE,
-                ),
+                os.path.join(tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE),
             )
         )
 
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        return {"url": urlpath}
+
     def _validate_config(self):
         # Validate Service Account configuration
         if (
@@ -207,9 +209,9 @@ class GDriveFileSystem(BaseFileSystem):
 
     @wrap_prop(threading.RLock())
     @cached_property
-    def _drive(self):
+    def fs(self):
         from pydrive2.auth import GoogleAuth
-        from pydrive2.drive import GoogleDrive
+        from pydrive2.fs import GDriveFileSystem as _GDriveFileSystem
 
         temporary_save_path = self._gdrive_user_credentials_path
         is_credentials_temp = os.getenv(
@@ -284,352 +286,19 @@ class GDriveFileSystem(BaseFileSystem):
             if is_credentials_temp:
                 os.remove(temporary_save_path)
 
-        return GoogleDrive(gauth)
-
-    @wrap_prop(threading.RLock())
-    @cached_property
-    def _ids_cache(self):
-        cache = {
-            "dirs": defaultdict(list),
-            "ids": {},
-            "root_id": self._get_item_id(
-                self.path_info,
-                use_cache=False,
-                hint="Confirm the directory exists and you can access it.",
-            ),
-        }
-
-        self._cache_path_id(self._path, cache["root_id"], cache=cache)
-
-        for item in self._gdrive_list(
-            "'{}' in parents and trashed=false".format(cache["root_id"])
-        ):
-            item_path = (self.path_info / item["title"]).path
-            self._cache_path_id(item_path, item["id"], cache=cache)
-
-        return cache
-
-    def _cache_path_id(self, path, *item_ids, cache=None):
-        cache = cache or self._ids_cache
-        for item_id in item_ids:
-            cache["dirs"][path].append(item_id)
-            cache["ids"][item_id] = path
-
-    @cached_property
-    def _list_params(self):
-        params = {"corpora": "default"}
-        if self._bucket != "root" and self._bucket != "appDataFolder":
-            drive_id = self._gdrive_shared_drive_id(self._bucket)
-            if drive_id:
-                logger.debug(
-                    "GDrive remote '{}' is using shared drive id '{}'.".format(
-                        self.path_info, drive_id
-                    )
-                )
-                params["driveId"] = drive_id
-                params["corpora"] = "drive"
-        return params
-
-    @_gdrive_retry
-    def _gdrive_shared_drive_id(self, item_id):
-        from pydrive2.files import ApiRequestError
-
-        param = {"id": item_id}
-        # it does not create a file on the remote
-        item = self._drive.CreateFile(param)
-        # ID of the shared drive the item resides in.
-        # Only populated for items in shared drives.
-        try:
-            item.FetchMetadata("driveId")
-        except ApiRequestError as exc:
-            error_code = exc.error.get("code", 0)
-            if error_code == 404:
-                raise DvcException(
-                    "'{}' for '{}':\n\n"
-                    "1. Confirm the directory exists and you can access it.\n"
-                    "2. Make sure that credentials in '{}'\n"
-                    "   are correct for this remote e.g. "
-                    "use the `gdrive_user_credentials_file` config\n"
-                    "   option if you use multiple GDrive remotes with "
-                    "different email accounts.\n\nDetails".format(
-                        item_id, self.path_info, self.credentials_location
-                    )
-                ) from exc
-            raise
-
-        return item.get("driveId", None)
-
-    @_gdrive_retry
-    def _gdrive_upload_fobj(self, fobj, parent_id, title):
-        item = self._drive.CreateFile(
-            {"title": title, "parents": [{"id": parent_id}]}
-        )
-        item.content = fobj
-        item.Upload()
-        return item
-
-    @_gdrive_retry
-    def _gdrive_download_file(
-        self, item_id, to_file, progress_desc, no_progress_bar
-    ):
-        param = {"id": item_id}
-        # it does not create a file on the remote
-        gdrive_file = self._drive.CreateFile(param)
-
-        with Tqdm(
-            desc=progress_desc,
-            disable=no_progress_bar,
-            bytes=True,
-            # explicit `bar_format` as `total` will be set by `update_to`
-            bar_format=Tqdm.BAR_FMT_DEFAULT,
-        ) as pbar:
-            gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
-
-    @contextmanager
-    @_gdrive_retry
-    def open(self, path_info, mode="r", encoding=None, **kwargs):
-        assert mode in {"r", "rt", "rb"}
-
-        item_id = self._get_item_id(path_info)
-        param = {"id": item_id}
-        # it does not create a file on the remote
-        gdrive_file = self._drive.CreateFile(param)
-        fd = gdrive_file.GetContentIOBuffer()
-        stream = IterStream(iter(fd))
-
-        if mode != "rb":
-            stream = io.TextIOWrapper(stream, encoding=encoding)
-
-        yield stream
-
-    @_gdrive_retry
-    def gdrive_delete_file(self, item_id):
-        from pydrive2.files import ApiRequestError
-
-        param = {"id": item_id}
-        # it does not create a file on the remote
-        item = self._drive.CreateFile(param)
-
-        try:
-            item.Trash() if self._trash_only else item.Delete()
-        except ApiRequestError as exc:
-            http_error_code = exc.error.get("code", 0)
-            if (
-                http_error_code == 403
-                and self._list_params["corpora"] == "drive"
-                and exc.GetField("location") == "file.permissions"
-            ):
-                raise DvcException(
-                    "Insufficient permissions to {}. You should have {} "
-                    "access level for the used shared drive. More details "
-                    "at {}.".format(
-                        "move the file into Trash"
-                        if self._trash_only
-                        else "permanently delete the file",
-                        "Manager or Content Manager"
-                        if self._trash_only
-                        else "Manager",
-                        "https://support.google.com/a/answer/7337554",
-                    )
-                ) from exc
-            raise
-
-    def _gdrive_list(self, query):
-        param = {"q": query, "maxResults": 1000}
-        param.update(self._list_params)
-        file_list = self._drive.ListFile(param)
-
-        # Isolate and decorate fetching of remote drive items in pages.
-        get_list = _gdrive_retry(lambda: next(file_list, None))
-
-        # Fetch pages until None is received, lazily flatten the thing.
-        return cat(iter(get_list, None))
-
-    @_gdrive_retry
-    def _gdrive_create_dir(self, parent_id, title):
-        parent = {"id": parent_id}
-        item = self._drive.CreateFile(
-            {"title": title, "parents": [parent], "mimeType": FOLDER_MIME_TYPE}
-        )
-        item.Upload()
-        return item
-
-    @wrap_with(threading.RLock())
-    def _create_dir(self, parent_id, title, remote_path):
-        cached = self._ids_cache["dirs"].get(remote_path)
-        if cached:
-            return cached[0]
-
-        item = self._gdrive_create_dir(parent_id, title)
-
-        if parent_id == self._ids_cache["root_id"]:
-            self._cache_path_id(remote_path, item["id"])
-
-        return item["id"]
-
-    def _get_remote_item_ids(self, parent_ids, title):
-        if not parent_ids:
-            return None
-        query = "trashed=false and ({})".format(
-            " or ".join(
-                f"'{parent_id}' in parents" for parent_id in parent_ids
-            )
-        )
-        query += " and title='{}'".format(title.replace("'", "\\'"))
-
-        # GDrive list API is case insensitive, we need to compare
-        # all results and pick the ones with the right title
-        return [
-            item["id"]
-            for item in self._gdrive_list(query)
-            if item["title"] == title
-        ]
-
-    def _get_cached_item_ids(self, path, use_cache):
-        if not path:
-            return [self._bucket]
-        if use_cache:
-            return self._ids_cache["dirs"].get(path, [])
-        return []
-
-    def _path_to_item_ids(self, path, create=False, use_cache=True):
-        item_ids = self._get_cached_item_ids(path, use_cache)
-        if item_ids:
-            return item_ids
-
-        parent_path, title = posixpath.split(path)
-        parent_ids = self._path_to_item_ids(parent_path, create, use_cache)
-        item_ids = self._get_remote_item_ids(parent_ids, title)
-        if item_ids:
-            return item_ids
-
-        return (
-            [self._create_dir(min(parent_ids), title, path)] if create else []
+        return _GDriveFileSystem(
+            self._with_bucket(self.path_info),
+            gauth,
+            trash_only=self._trash_only,
         )
 
-    def _get_item_id(self, path_info, create=False, use_cache=True, hint=None):
-        assert path_info.bucket == self._bucket
+    def _with_bucket(self, path):
+        if isinstance(path, self.PATH_CLS):
+            return posixpath.join(path.bucket, path.path)
 
-        item_ids = self._path_to_item_ids(path_info.path, create, use_cache)
-        if item_ids:
-            return min(item_ids)
+        return super()._with_bucket(path)
 
-        assert not create
-        raise FileMissingError(path_info, hint)
-
-    def exists(self, path_info, use_dvcignore=True):
-        try:
-            self._get_item_id(path_info)
-        except FileMissingError:
-            return False
-        else:
-            return True
-
-    def _gdrive_list_ids(self, query_ids):
-        query = " or ".join(
-            f"'{query_id}' in parents" for query_id in query_ids
-        )
-        query = f"({query}) and trashed=false"
-        return self._gdrive_list(query)
-
-    def find(self, path_info, detail=False):
-        root_path = path_info.path
-        seen_paths = set()
-
-        dir_ids = [self._ids_cache["ids"].copy()]
-        while dir_ids:
-            query_ids = {
-                dir_id: dir_name
-                for dir_id, dir_name in dir_ids.pop().items()
-                if posixpath.commonpath([root_path, dir_name]) == root_path
-                if dir_id not in seen_paths
-            }
-            if not query_ids:
-                continue
-
-            seen_paths |= query_ids.keys()
-
-            new_query_ids = {}
-            dir_ids.append(new_query_ids)
-            for item in self._gdrive_list_ids(query_ids):
-                parent_id = item["parents"][0]["id"]
-                item_path = posixpath.join(query_ids[parent_id], item["title"])
-                if item["mimeType"] == FOLDER_MIME_TYPE:
-                    new_query_ids[item["id"]] = item_path
-                    self._cache_path_id(item_path, item["id"])
-                    continue
-
-                if detail:
-                    yield {
-                        "type": "file",
-                        "name": item_path,
-                        "size": item["fileSize"],
-                        "checksum": item["md5Checksum"],
-                    }
-                else:
-                    yield item_path
-
-    def ls(self, path_info, detail=False):
-        cached = path_info.path in self._ids_cache["dirs"]
-        if cached:
-            dir_ids = self._ids_cache["dirs"][path_info.path]
-        else:
-            dir_ids = self._path_to_item_ids(path_info.path)
-
-        if not dir_ids:
-            return None
-
-        root_path = path_info.path
-        for item in self._gdrive_list_ids(dir_ids):
-            item_path = posixpath.join(root_path, item["title"])
-            if detail:
-                if item["mimeType"] == FOLDER_MIME_TYPE:
-                    yield {"type": "directory", "name": item_path}
-                else:
-                    yield {
-                        "type": "file",
-                        "name": item_path,
-                        "size": item["fileSize"],
-                        "checksum": item["md5Checksum"],
-                    }
-            else:
-                yield item_path
-
-        if not cached:
-            self._cache_path_id(root_path, *dir_ids)
-
-    def walk_files(self, path_info, **kwargs):
-        for file in self.find(path_info):
-            yield path_info.replace(path=file)
-
-    def remove(self, path_info):
-        item_id = self._get_item_id(path_info)
-        self.gdrive_delete_file(item_id)
-
-    def info(self, path_info):
-        item_id = self._get_item_id(path_info)
-        gdrive_file = self._drive.CreateFile({"id": item_id})
-        gdrive_file.FetchMetadata(fields="fileSize")
-        return {"size": gdrive_file.get("fileSize")}
-
-    def _upload_fobj(self, fobj, to_info):
-        dirname = to_info.parent
-        assert dirname
-        parent_id = self._get_item_id(dirname, create=True)
-        self._gdrive_upload_fobj(fobj, parent_id, to_info.name)
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
-    ):
-        with open(from_file, "rb") as fobj:
-            self.upload_fobj(
-                fobj,
-                to_info,
-                no_progress_bar=no_progress_bar,
-                desc=name or to_info.name,
-                total=os.path.getsize(from_file),
-            )
-
-    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        item_id = self._get_item_id(from_info)
-        self._gdrive_download_file(item_id, to_file, name, no_progress_bar)
+    def _upload_fobj(self, fobj, to_info, size: int = None):
+        rpath = self._with_bucket(to_info)
+        self.makedirs(os.path.dirname(rpath))
+        return self.fs.upload_fobj(fobj, rpath, size=size)

dvc/fs/git.py

@@ -1,8 +1,6 @@
 import errno
 import os
 
-from funcy import cached_property
-
 from dvc.utils import is_exec, relpath
 
 from .base import BaseFileSystem
@@ -11,46 +9,34 @@ from .base import BaseFileSystem
 class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
-    def __init__(
-        self, root_dir, trie, use_dvcignore=False, dvcignore_root=None
-    ):
-        super().__init__(None, {})
-        self._fs_root = root_dir
+    sep = os.sep
+
+    scheme = "local"
+
+    def __init__(self, root_dir, trie):
+        super().__init__()
+        self._root = root_dir
         self.trie = trie
-        self.use_dvcignore = use_dvcignore
-        self.dvcignore_root = dvcignore_root
 
     @property
     def rev(self):
         return self.trie.rev
 
-    @property
-    def fs_root(self):
-        return self._fs_root
-
     def _get_key(self, path):
         if isinstance(path, str):
             if not os.path.isabs(path):
                 relparts = path.split(os.sep)
             else:
-                relparts = relpath(path, self.fs_root).split(os.sep)
+                relparts = relpath(path, self._root).split(os.sep)
         else:
-            relparts = path.relative_to(self.fs_root).parts
+            relparts = path.relative_to(self._root).parts
         if relparts == ["."]:
             return ()
         return tuple(relparts)
 
-    @cached_property
-    def dvcignore(self):
-        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
-
-        root = self.dvcignore_root or self.fs_root
-        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
-        return cls(self, root)
-
     def open(
-        self, path, mode="r", encoding=None
-    ):  # pylint: disable=arguments-differ
+        self, path, mode="r", encoding=None, **kwargs
+    ):  # pylint: disable=arguments-renamed
         # NOTE: this is incorrect, we need to use locale to determine default
         # encoding.
         encoding = encoding or "utf-8"
@@ -66,50 +52,27 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 errno.EISDIR, os.strerror(errno.EISDIR), path
             ) from exc
 
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        def _is_ignored(path):
-            return self.dvcignore.is_ignored_file(
-                path
-            ) or self.dvcignore.is_ignored_dir(path)
-
-        if use_dvcignore and _is_ignored(path):
-            return False
-
-        key = self._get_key(path)
+    def exists(self, path_info) -> bool:
+        key = self._get_key(path_info)
         return self.trie.exists(key)
 
-    def isdir(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        if use_dvcignore and self.dvcignore.is_ignored_dir(path):
-            return False
-        key = self._get_key(path)
+    def isdir(self, path_info) -> bool:
+        key = self._get_key(path_info)
         return self.trie.isdir(key)
 
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        if self.dvcignore.is_ignored_file(path):
-            return False
-        key = self._get_key(path)
+    def isfile(self, path_info) -> bool:
+        key = self._get_key(path_info)
         return self.trie.isfile(key)
 
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        use_dvcignore=True,
-        ignore_subrepos=True,
-    ):
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
         - no support for symlinks
         """
-        if not self.isdir(top, use_dvcignore=use_dvcignore):
+        if not self.isdir(top):
             if onerror:
-                if self.exists(top, use_dvcignore=use_dvcignore):
+                if self.exists(top):
                     exc = NotADirectoryError(
                         errno.ENOTDIR, os.strerror(errno.ENOTDIR), top
                     )
@@ -123,46 +86,46 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         key = self._get_key(top)
         for prefix, dirs, files in self.trie.walk(key, topdown=topdown):
             if prefix:
-                root = os.path.join(self.fs_root, os.sep.join(prefix))
+                root = os.path.join(self._root, os.sep.join(prefix))
             else:
-                root = self.fs_root
-            if use_dvcignore:
-                dirs[:], files[:] = self.dvcignore(
-                    root, dirs, files, ignore_subrepos=ignore_subrepos,
-                )
+                root = self._root
             yield root, dirs, files
 
     def isexec(self, path_info):
         if not self.exists(path_info):
             return False
 
-        mode = self.stat(path_info).st_mode
+        mode = self.info(path_info)["mode"]
         return is_exec(mode)
 
     def info(self, path_info):
         key = self._get_key(path_info)
         try:
-            st = self.trie.stat(key)
+            return self.trie.info(key)
         except KeyError:
             raise FileNotFoundError(
                 errno.ENOENT, os.strerror(errno.ENOENT), path_info
             )
-        return {"size": st.st_size}
 
-    def stat(self, path):
-        key = self._get_key(path)
-        try:
-            return self.trie.stat(key)
-        except KeyError:
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), path
-            )
+    def checksum(self, path_info):
+        return self.info(path_info)["sha"]
 
-    def walk_files(self, top):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top):
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info, **kwargs):
             for file in files:
                 # NOTE: os.path.join is ~5.5 times slower
                 yield f"{root}{os.sep}{file}"
 
-    def _reset(self):
-        return self.__dict__.pop("dvcignore", None)
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **kwargs
+    ):
+        import shutil
+
+        from dvc.progress import Tqdm
+
+        with open(to_file, "wb+") as to_fobj:
+            with Tqdm.wrapattr(
+                to_fobj, "write", desc=name, disable=no_progress_bar
+            ) as wrapped:
+                with self.open(from_info, "rb", **kwargs) as from_fobj:
+                    shutil.copyfileobj(from_fobj, wrapped)

dvc/fs/gs.py

@@ -6,23 +6,19 @@ from funcy import cached_property, wrap_prop
 from dvc.path_info import CloudURLInfo
 from dvc.scheme import Schemes
 
-from .fsspec_wrapper import FSSpecWrapper
+from .fsspec_wrapper import ObjectFSWrapper
 
 
-class GSFileSystem(FSSpecWrapper):
+# pylint:disable=abstract-method
+class GSFileSystem(ObjectFSWrapper):
     scheme = Schemes.GS
     PATH_CLS = CloudURLInfo
     REQUIRES = {"gcsfs": "gcsfs"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
+    TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-
-        url = config.get("url", "gs://")
-        self.path_info = self.PATH_CLS(url)
-
-    def _prepare_credentials(self, config):
+    def _prepare_credentials(self, **config):
         login_info = {"consistency": None}
         login_info["project"] = config.get("projectname")
         login_info["token"] = config.get("credentialpath")

dvc/fs/hdfs.py

@@ -1,3 +1,4 @@
+import errno
 import io
 import logging
 import os
@@ -6,7 +7,6 @@ import shutil
 import subprocess
 from collections import deque
 from contextlib import closing, contextmanager
-from urllib.parse import urlparse
 
 from dvc.hash_info import HashInfo
 from dvc.progress import Tqdm
@@ -65,27 +65,24 @@ class HDFSFileSystem(BaseFileSystem):
     PARAM_CHECKSUM = "checksum"
     TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    BLOCK_SIZE = 2 ** 26  # 128MB
 
-        self.path_info = None
-        url = config.get("url")
-        if not url:
-            return
+    def __init__(self, **config):
+        super().__init__(**config)
 
-        parsed = urlparse(url)
-        user = parsed.username or config.get("user")
-
-        self.path_info = self.PATH_CLS.from_parts(
-            scheme=self.scheme,
-            host=parsed.hostname,
-            user=user,
-            port=parsed.port,
-            path=parsed.path,
-        )
+        self.host = config["host"]
+        self.user = config.get("user")
+        self.port = config.get("port")
 
         self.kerb_ticket = config.get("kerb_ticket")
 
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        from fsspec.implementations.hdfs import PyArrowHDFS
+
+        # pylint:disable=protected-access
+        return PyArrowHDFS._get_kwargs_from_urls(urlpath)
+
     def hdfs(self, path_info):
         import pyarrow.fs
 
@@ -123,7 +120,7 @@ class HDFSFileSystem(BaseFileSystem):
                 raise FileNotFoundError(*e.args)
             raise
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         assert not isinstance(path_info, list)
         assert path_info.scheme == "hdfs"
         with self.hdfs(path_info) as hdfs:
@@ -159,15 +156,15 @@ class HDFSFileSystem(BaseFileSystem):
         if not topdown:
             yield root, dirs, nondirs
 
-    def walk(self, path_info, **kwargs):
-        if not self.isdir(path_info):
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        if not self.isdir(top):
             return
 
-        with self.hdfs(path_info) as hdfs:
+        with self.hdfs(top) as hdfs:
             for root, dnames, fnames in self._walk(
-                hdfs, path_info.path, **kwargs
+                hdfs, top.path, topdown=topdown
             ):
-                yield path_info.replace(path=root), dnames, fnames
+                yield top.replace(path=root), dnames, fnames
 
     def walk_files(self, path_info, **kwargs):
         for root, _, fnames in self.walk(path_info):
@@ -188,7 +185,7 @@ class HDFSFileSystem(BaseFileSystem):
                 else:
                     hdfs.delete_file(path_info.path)
 
-    def makedirs(self, path_info):
+    def makedirs(self, path_info, **kwargs):
         with self.hdfs(path_info) as hdfs:
             # NOTE: fs.create_dir creates parents by default
             hdfs.create_dir(path_info.path)
@@ -226,8 +223,19 @@ class HDFSFileSystem(BaseFileSystem):
 
     def info(self, path_info):
         with self.hdfs(path_info) as hdfs:
-            finfo = hdfs.get_file_info(path_info.path)
-            return {"size": finfo.size}
+            from pyarrow.fs import FileType
+
+            file_info = hdfs.get_file_info(path_info.path)
+            if file_info.type is FileType.Directory:
+                kind = "directory"
+            elif file_info.type is FileType.File:
+                kind = "file"
+            else:
+                raise FileNotFoundError(
+                    errno.ENOENT, os.strerror(errno.ENOENT), path_info
+                )
+
+            return {"size": file_info.size, "type": kind}
 
     def checksum(self, path_info):
         return HashInfo(
@@ -236,10 +244,10 @@ class HDFSFileSystem(BaseFileSystem):
             size=self.getsize(path_info),
         )
 
-    def _upload_fobj(self, fobj, to_info):
+    def _upload_fobj(self, fobj, to_info, **kwargs):
         with self.hdfs(to_info) as hdfs:
             with hdfs.open_output_stream(to_info.path) as fdest:
-                shutil.copyfileobj(fobj, fdest)
+                shutil.copyfileobj(fobj, fdest, self.BLOCK_SIZE)
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
@@ -256,7 +264,7 @@ class HDFSFileSystem(BaseFileSystem):
                     disable=no_progress_bar,
                 ) as wrapped:
                     with hdfs.open_output_stream(tmp_file) as sobj:
-                        sobj.write(wrapped.read())
+                        shutil.copyfileobj(wrapped, sobj, self.BLOCK_SIZE)
             hdfs.move(tmp_file, to_info.path)
 
     def _download(
@@ -274,4 +282,4 @@ class HDFSFileSystem(BaseFileSystem):
                     disable=no_progress_bar,
                 ) as wrapped:
                     with hdfs.open_input_stream(from_info.path) as sobj:
-                        wrapped.write(sobj.read())
+                        shutil.copyfileobj(sobj, wrapped, self.BLOCK_SIZE)

dvc/fs/http.py

@@ -1,19 +1,12 @@
-import logging
-import os.path
 import threading
-from typing import Optional
 
-from funcy import cached_property, memoize, wrap_prop, wrap_with
+from funcy import cached_property, memoize, wrap_with
 
-import dvc.prompt as prompt
-from dvc.exceptions import DvcException, HTTPError
+from dvc import prompt
 from dvc.path_info import HTTPURLInfo
-from dvc.progress import Tqdm
 from dvc.scheme import Schemes
 
-from .base import BaseFileSystem
-
-logger = logging.getLogger(__name__)
+from .fsspec_wrapper import CallbackMixin, FSSpecWrapper, NoDirectoriesMixin
 
 
 @wrap_with(threading.Lock())
@@ -25,176 +18,98 @@ def ask_password(host, user):
     )
 
 
-class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
+def make_context(ssl_verify):
+    if isinstance(ssl_verify, bool) or ssl_verify is None:
+        return ssl_verify
+
+    # If this is a path, then we will create an
+    # SSL context for it, and load the given certificate.
+    import ssl
+
+    context = ssl.create_default_context()
+    context.load_verify_locations(ssl_verify)
+    return context
+
+
+# pylint: disable=abstract-method
+class HTTPFileSystem(CallbackMixin, NoDirectoriesMixin, FSSpecWrapper):
     scheme = Schemes.HTTP
     PATH_CLS = HTTPURLInfo
-    PARAM_CHECKSUM = "etag"
+    PARAM_CHECKSUM = "checksum"
+    REQUIRES = {"aiohttp": "aiohttp", "aiohttp-retry": "aiohttp_retry"}
     CAN_TRAVERSE = False
 
     SESSION_RETRIES = 5
     SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 60
-    CHUNK_SIZE = 2 ** 16
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def _prepare_credentials(self, **config):
+        import aiohttp
+        from fsspec.asyn import fsspec_loop
 
-        url = config.get("url")
-        if url:
-            self.path_info = self.PATH_CLS(url)
-            self.user = config.get("user", None)
-            self.host = self.path_info.host
-        else:
-            self.path_info = None
+        from dvc.config import ConfigError
 
-        self.auth = config.get("auth", None)
-        self.custom_auth_header = config.get("custom_auth_header", None)
-        self.password = config.get("password", None)
-        self.ask_password = config.get("ask_password", False)
-        self.headers = {}
-        self.ssl_verify = config.get("ssl_verify", True)
-        self.method = config.get("method", "POST")
+        credentials = {}
+        client_args = credentials.setdefault("client_args", {})
 
-    def _auth_method(self):
-        from requests.auth import HTTPBasicAuth, HTTPDigestAuth
+        if config.get("auth"):
+            user = config.get("user")
+            password = config.get("password")
+            custom_auth_header = config.get("custom_auth_header")
 
-        if self.auth:
-            if self.ask_password and self.password is None:
-                self.password = ask_password(self.host, self.user)
-            if self.auth == "basic":
-                return HTTPBasicAuth(self.user, self.password)
-            if self.auth == "digest":
-                return HTTPDigestAuth(self.user, self.password)
-            if self.auth == "custom" and self.custom_auth_header:
-                self.headers.update({self.custom_auth_header: self.password})
-        return None
+            if password is None and config.get("ask_password"):
+                password = ask_password(config.get("url"), user or "custom")
 
-    def _generate_download_url(self, path_info):
-        return path_info.url
+            auth_method = config["auth"]
+            if auth_method == "basic":
+                if user is None or password is None:
+                    raise ConfigError(
+                        "HTTP 'basic' authentication require both "
+                        "'user' and 'password'"
+                    )
 
-    @wrap_prop(threading.Lock())
-    @cached_property
-    def _session(self):
-        import requests
-        from requests.adapters import HTTPAdapter
-        from urllib3.util.retry import Retry
+                client_args["auth"] = aiohttp.BasicAuth(user, password)
+            elif auth_method == "custom":
+                if custom_auth_header is None or password is None:
+                    raise ConfigError(
+                        "HTTP 'custom' authentication require both "
+                        "'custom_auth_header' and 'password'"
+                    )
+                credentials["headers"] = {custom_auth_header: password}
+            else:
+                raise NotImplementedError(
+                    f"Auth method {auth_method!r} is not supported."
+                )
 
-        session = requests.Session()
+        if "ssl_verify" in config:
+            with fsspec_loop():
+                client_args["connector"] = aiohttp.TCPConnector(
+                    ssl=make_context(config["ssl_verify"])
+                )
 
-        session.verify = self.ssl_verify
+        credentials["get_client"] = self.get_client
+        self.upload_method = config.get("method", "POST")
+        return credentials
 
-        retries = Retry(
-            total=self.SESSION_RETRIES,
-            backoff_factor=self.SESSION_BACKOFF_FACTOR,
+    async def get_client(self, **kwargs):
+        from aiohttp_retry import ExponentialRetry, RetryClient
+
+        kwargs["retry_options"] = ExponentialRetry(
+            attempts=self.SESSION_RETRIES,
+            factor=self.SESSION_BACKOFF_FACTOR,
+            max_timeout=self.REQUEST_TIMEOUT,
         )
 
-        session.mount("http://", HTTPAdapter(max_retries=retries))
-        session.mount("https://", HTTPAdapter(max_retries=retries))
+        return RetryClient(**kwargs)
 
-        return session
+    @cached_property
+    def fs(self):
+        from fsspec.implementations.http import (
+            HTTPFileSystem as _HTTPFileSystem,
+        )
 
-    def request(self, method, url, **kwargs):
-        import requests
+        return _HTTPFileSystem(timeout=self.REQUEST_TIMEOUT)
 
-        kwargs.setdefault("allow_redirects", True)
-        kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
-
-        try:
-            res = self._session.request(
-                method,
-                url,
-                auth=self._auth_method(),
-                headers=self.headers,
-                **kwargs,
-            )
-
-            redirect_no_location = (
-                kwargs["allow_redirects"]
-                and res.status_code in (301, 302)
-                and "location" not in res.headers
-            )
-
-            if redirect_no_location:
-                # AWS s3 doesn't like to add a location header to its redirects
-                # from https://s3.amazonaws.com/<bucket name>/* type URLs.
-                # This should be treated as an error
-                raise requests.exceptions.RequestException
-
-            return res
-
-        except requests.exceptions.RequestException:
-            raise DvcException(f"could not perform a {method} request")
-
-    def _head(self, url):
-        response = self.request("HEAD", url)
-        if response.ok:
-            return response
-
-        # Sometimes servers are configured to forbid HEAD requests
-        # Context: https://github.com/iterative/dvc/issues/4131
-        with self.request("GET", url, stream=True) as r:
-            if r.ok:
-                return r
-
-        return response
-
-    def exists(self, path_info, use_dvcignore=True):
-        res = self._head(path_info.url)
-        if res.status_code == 404:
-            return False
-        if bool(res):
-            return True
-        raise HTTPError(res.status_code, res.reason)
-
-    def info(self, path_info):
-        resp = self._head(path_info.url)
-        etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5")
-        size = self._content_length(resp)
-        return {"etag": etag, "size": size}
-
-    def _upload_fobj(self, fobj, to_info):
-        def chunks(fobj):
-            while True:
-                chunk = fobj.read(self.CHUNK_SIZE)
-                if not chunk:
-                    break
-                yield chunk
-
-        response = self.request(self.method, to_info.url, data=chunks(fobj))
-        if response.status_code not in (200, 201):
-            raise HTTPError(response.status_code, response.reason)
-
-    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        response = self.request("GET", from_info.url, stream=True)
-        if response.status_code != 200:
-            raise HTTPError(response.status_code, response.reason)
-        with open(to_file, "wb") as fd:
-            with Tqdm.wrapattr(
-                fd,
-                "write",
-                total=None
-                if no_progress_bar
-                else self._content_length(response),
-                leave=False,
-                desc=from_info.url if name is None else name,
-                disable=no_progress_bar,
-            ) as fd_wrapped:
-                for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
-                    fd_wrapped.write(chunk)
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
-    ):
-        with open(from_file, "rb") as fobj:
-            self.upload_fobj(
-                fobj,
-                to_info,
-                no_progress_bar=no_progress_bar,
-                desc=name or to_info.url,
-                total=None if no_progress_bar else os.path.getsize(from_file),
-            )
-
-    @staticmethod
-    def _content_length(response) -> Optional[int]:
-        res = response.headers.get("Content-Length")
-        return int(res) if res else None
+    def _entry_hook(self, entry):
+        entry["checksum"] = entry.get("ETag") or entry.get("Content-MD5")
+        return entry

dvc/fs/local.py

@@ -1,8 +1,5 @@
 import logging
 import os
-import stat
-
-from funcy import cached_property
 
 from dvc.path_info import PathInfo
 from dvc.scheme import Schemes
@@ -16,76 +13,45 @@ logger = logging.getLogger(__name__)
 
 
 class LocalFileSystem(BaseFileSystem):
+    sep = os.sep
+
     scheme = Schemes.LOCAL
     PATH_CLS = PathInfo
     PARAM_CHECKSUM = "md5"
     PARAM_PATH = "path"
     TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config, use_dvcignore=False, dvcignore_root=None):
-        super().__init__(repo, config)
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url) if url else None
-        self.use_dvcignore = use_dvcignore
-        self.dvcignore_root = dvcignore_root
+    def __init__(self, **config):
+        from fsspec.implementations.local import LocalFileSystem as LocalFS
 
-    @property
-    def fs_root(self):
-        return self.config.get("url")
-
-    @cached_property
-    def dvcignore(self):
-        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
-
-        root = self.dvcignore_root or self.fs_root
-        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
-        return cls(self, root)
+        super().__init__(**config)
+        self.fs = LocalFS()
 
     @staticmethod
     def open(path_info, mode="r", encoding=None, **kwargs):
         return open(path_info, mode=mode, encoding=encoding)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         assert isinstance(path_info, str) or path_info.scheme == "local"
-        if self.repo:
-            ret = os.path.lexists(path_info)
-        else:
-            ret = os.path.exists(path_info)
-        if not ret:
-            return False
-        if not use_dvcignore:
-            return True
+        # TODO: replace this with os.path.exists once the problem is fixed on
+        # the fsspec https://github.com/intake/filesystem_spec/issues/742
+        return os.path.lexists(path_info)
 
-        return not self.dvcignore.is_ignored_file(
-            path_info
-        ) and not self.dvcignore.is_ignored_dir(path_info)
+    def checksum(self, path_info) -> str:
+        return self.fs.checksum(path_info)
 
-    def isfile(self, path_info):
-        if not os.path.isfile(path_info):
-            return False
+    def isfile(self, path_info) -> bool:
+        return os.path.isfile(path_info)
 
-        return not self.dvcignore.is_ignored_file(path_info)
-
-    def isdir(
-        self, path_info, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        if not os.path.isdir(path_info):
-            return False
-        return not (use_dvcignore and self.dvcignore.is_ignored_dir(path_info))
+    def isdir(self, path_info) -> bool:
+        return os.path.isdir(path_info)
 
     def iscopy(self, path_info):
         return not (
             System.is_symlink(path_info) or System.is_hardlink(path_info)
         )
 
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        use_dvcignore=True,
-        ignore_subrepos=True,
-    ):
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Directory fs generator.
 
         See `os.walk` for the docs. Differences:
@@ -94,14 +60,6 @@ class LocalFileSystem(BaseFileSystem):
         for root, dirs, files in os.walk(
             top, topdown=topdown, onerror=onerror
         ):
-            if use_dvcignore:
-                dirs[:], files[:] = self.dvcignore(
-                    os.path.abspath(root),
-                    dirs,
-                    files,
-                    ignore_subrepos=ignore_subrepos,
-                )
-
             yield os.path.normpath(root), dirs, files
 
     def walk_files(self, path_info, **kwargs):
@@ -125,21 +83,17 @@ class LocalFileSystem(BaseFileSystem):
                 raise NotImplementedError
         remove(path_info)
 
-    def makedirs(self, path_info):
-        makedirs(path_info, exist_ok=True)
+    def makedirs(self, path_info, **kwargs):
+        makedirs(path_info, exist_ok=kwargs.pop("exist_ok", True))
 
     def isexec(self, path_info):
-        mode = self.stat(path_info).st_mode
+        mode = self.info(path_info)["mode"]
         return is_exec(mode)
 
-    def stat(self, path):
-        if self.dvcignore.is_ignored(path):
-            raise FileNotFoundError
-
-        return os.stat(path)
-
     def move(self, from_info, to_info):
-        if from_info.scheme != "local" or to_info.scheme != "local":
+        if (
+            isinstance(from_info, PathInfo) and from_info.scheme != "local"
+        ) or (isinstance(to_info, PathInfo) and to_info.scheme != "local"):
             raise NotImplementedError
 
         self.makedirs(to_info.parent)
@@ -154,7 +108,7 @@ class LocalFileSystem(BaseFileSystem):
             self.remove(tmp_info)
             raise
 
-    def _upload_fobj(self, fobj, to_info):
+    def _upload_fobj(self, fobj, to_info, **kwargs):
         self.makedirs(to_info.parent)
         tmp_info = to_info.parent / tmp_fname("")
         try:
@@ -201,16 +155,11 @@ class LocalFileSystem(BaseFileSystem):
     def reflink(self, from_info, to_info):
         System.reflink(from_info, to_info)
 
-    @staticmethod
-    def info(path_info):
-        st = os.stat(path_info)
-        return {
-            "size": st.st_size,
-            "type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
-        }
+    def info(self, path_info):
+        return self.fs.info(path_info)
 
     def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs,
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
         makedirs(to_info.parent, exist_ok=True)
 
@@ -227,6 +176,3 @@ class LocalFileSystem(BaseFileSystem):
         copyfile(
             from_info, to_file, no_progress_bar=no_progress_bar, name=name
         )
-
-    def _reset(self):
-        return self.__dict__.pop("dvcignore", None)

dvc/fs/memory.py

@@ -1,34 +1,35 @@
-from .base import BaseFileSystem
+import threading
+
+from funcy import cached_property, wrap_prop
+
+from dvc.path_info import CloudURLInfo
+from dvc.scheme import Schemes
+
+from .fsspec_wrapper import FSSpecWrapper
 
 
-class MemoryFileSystem(BaseFileSystem):
-    scheme = "local"
+class MemoryFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
+    scheme = Schemes.MEMORY
     PARAM_CHECKSUM = "md5"
+    PATH_CLS = CloudURLInfo
+    TRAVERSE_PREFIX_LEN = 2
+    DEFAULT_BLOCKSIZE = 4096
 
-    def __init__(self, repo, config):
+    def __eq__(self, other):
+        # NOTE: all fsspec MemoryFileSystem instances are equivalent and use a
+        # single global store
+        return isinstance(other, type(self))
+
+    __hash__ = FSSpecWrapper.__hash__
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def fs(self):
         from fsspec.implementations.memory import MemoryFileSystem as MemFS
 
-        super().__init__(repo, config)
+        return MemFS(**self.fs_args)
 
-        self.fs = MemFS()
-
-    def exists(self, path_info, use_dvcignore=True):
-        return self.fs.exists(path_info.fspath)
-
-    def open(self, path_info, mode="r", encoding=None, **kwargs):
-        return self.fs.open(
-            path_info.fspath, mode=mode, encoding=encoding, **kwargs
-        )
-
-    def info(self, path_info):
-        return self.fs.info(path_info.fspath)
-
-    def stat(self, path_info):
-        import os
-
-        info = self.fs.info(path_info.fspath)
-
-        return os.stat_result((0, 0, 0, 0, 0, 0, info["size"], 0, 0, 0))
-
-    def walk_files(self, path_info, **kwargs):
-        raise NotImplementedError
+    def open(self, *args, **kwargs):
+        with super().open(*args, **kwargs) as fobj:
+            fobj.blocksize = self.DEFAULT_BLOCKSIZE
+            return fobj

dvc/fs/oss.py

@@ -5,137 +5,37 @@ import threading
 from funcy import cached_property, wrap_prop
 
 from dvc.path_info import CloudURLInfo
-from dvc.progress import Tqdm
 from dvc.scheme import Schemes
 
-from .base import BaseFileSystem
+from .fsspec_wrapper import CallbackMixin, ObjectFSWrapper
 
 logger = logging.getLogger(__name__)
 
 
-class OSSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
-    """
-    oss2 document:
-    https://www.alibabacloud.com/help/doc-detail/32026.htm
-
-
-    Examples
-    ----------
-    $ dvc remote add myremote oss://my-bucket/path
-    Set key id, key secret and endpoint using modify command
-    $ dvc remote modify myremote oss_key_id my-key-id
-    $ dvc remote modify myremote oss_key_secret my-key-secret
-    $ dvc remote modify myremote oss_endpoint endpoint
-    or environment variables
-    $ export OSS_ACCESS_KEY_ID="my-key-id"
-    $ export OSS_ACCESS_KEY_SECRET="my-key-secret"
-    $ export OSS_ENDPOINT="endpoint"
-    """
-
+# pylint:disable=abstract-method
+class OSSFileSystem(CallbackMixin, ObjectFSWrapper):
     scheme = Schemes.OSS
     PATH_CLS = CloudURLInfo
-    REQUIRES = {"oss2": "oss2"}
+    REQUIRES = {"ossfs": "ossfs"}
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 100
+    DETAIL_FIELDS = frozenset(("etag", "size"))
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url) if url else None
-
-        self.bucket = self.path_info.bucket
-        self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
-
-        self.key_id = (
-            config.get("oss_key_id")
-            or os.getenv("OSS_ACCESS_KEY_ID")
-            or "defaultId"
+    def _prepare_credentials(self, **config):
+        login_info = {}
+        login_info["key"] = config.get("oss_key_id") or os.getenv(
+            "OSS_ACCESS_KEY_ID"
         )
-
-        self.key_secret = (
-            config.get("oss_key_secret")
-            or os.getenv("OSS_ACCESS_KEY_SECRET")
-            or "defaultSecret"
+        login_info["secret"] = config.get("oss_key_secret") or os.getenv(
+            "OSS_ACCESS_KEY_SECRET"
         )
+        login_info["endpoint"] = config.get("oss_endpoint")
+        return login_info
 
     @wrap_prop(threading.Lock())
     @cached_property
-    def oss_service(self):
-        import oss2
+    def fs(self):
+        from ossfs import OSSFileSystem as _OSSFileSystem
 
-        logger.debug(f"key id: {self.key_id}")
-        logger.debug(f"key secret: {self.key_secret}")
-
-        auth = oss2.Auth(self.key_id, self.key_secret)
-        bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
-
-        # Ensure bucket exists
-        try:
-            bucket.get_bucket_info()
-        except oss2.exceptions.NoSuchBucket:
-            bucket.create_bucket(
-                oss2.BUCKET_ACL_PUBLIC_READ,
-                oss2.models.BucketCreateConfig(
-                    oss2.BUCKET_STORAGE_CLASS_STANDARD
-                ),
-            )
-        return bucket
-
-    def _generate_download_url(self, path_info, expires=3600):
-        assert path_info.bucket == self.bucket
-
-        return self.oss_service.sign_url("GET", path_info.path, expires)
-
-    def exists(self, path_info, use_dvcignore=True):
-        paths = self._list_paths(path_info)
-        return any(path_info.path == path for path in paths)
-
-    def _list_paths(self, path_info):
-        import oss2
-
-        for blob in oss2.ObjectIterator(
-            self.oss_service, prefix=path_info.path
-        ):
-            yield blob.key
-
-    def walk_files(self, path_info, **kwargs):
-        if not kwargs.pop("prefix", False):
-            path_info = path_info / ""
-        for fname in self._list_paths(path_info):
-            if fname.endswith("/"):
-                continue
-
-            yield path_info.replace(path=fname)
-
-    def remove(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        logger.debug(f"Removing oss://{path_info}")
-        self.oss_service.delete_object(path_info.path)
-
-    def _upload_fobj(self, fobj, to_info):
-        self.oss_service.put_object(to_info.path, fobj)
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
-    ):
-        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.oss_service.put_object_from_file(
-                to_info.path, from_file, progress_callback=pbar.update_to
-            )
-
-    def _download(
-        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
-    ):
-        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            import oss2
-
-            oss2.resumable_download(
-                self.oss_service,
-                from_info.path,
-                to_file,
-                progress_callback=pbar.update_to,
-            )
+        return _OSSFileSystem(**self.fs_args)

dvc/fs/repo.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import stat
 import threading
 from contextlib import suppress
 from itertools import takewhile
@@ -31,16 +30,33 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
     """
 
+    sep = os.sep
+
     scheme = "local"
     PARAM_CHECKSUM = "md5"
+    PARAM_REPO_URL = "repo_url"
+    PARAM_REPO_ROOT = "repo_root"
+    PARAM_REV = "rev"
+    PARAM_CACHE_DIR = "cache_dir"
+    PARAM_CACHE_TYPES = "cache_types"
+    PARAM_SUBREPOS = "subrepos"
 
     def __init__(
-        self, repo, subrepos=False, repo_factory: RepoFactory = None,
+        self,
+        repo: Optional["Repo"] = None,
+        subrepos=False,
+        repo_factory: RepoFactory = None,
+        **kwargs,
     ):
-        super().__init__(repo, {"url": repo.root_dir})
+        super().__init__()
 
         from dvc.utils.collections import PathStringTrie
 
+        if repo is None:
+            repo, repo_factory = self._repo_from_fs_config(
+                subrepos=subrepos, **kwargs
+            )
+
         if not repo_factory:
             from dvc.repo import Repo
 
@@ -49,6 +65,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             self.repo_factory = repo_factory
 
         self._main_repo = repo
+        self.hash_jobs = repo.fs.hash_jobs
         self.root_dir = repo.root_dir
         self._traverse_subrepos = subrepos
 
@@ -61,9 +78,76 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         """Keep a dvcfs instance of each repo."""
 
         if hasattr(repo, "dvc_dir"):
-            self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
+            self._dvcfss[repo.root_dir] = DvcFileSystem(repo=repo)
 
-    def _get_repo(self, path) -> Optional["Repo"]:
+    @property
+    def repo_url(self):
+        if self._main_repo is None:
+            return None
+        return self._main_repo.url
+
+    @property
+    def config(self):
+        return {
+            self.PARAM_REPO_URL: self.repo_url,
+            self.PARAM_REPO_ROOT: self.root_dir,
+            self.PARAM_REV: getattr(self._main_repo.fs, "rev", None),
+            self.PARAM_CACHE_DIR: os.path.abspath(
+                self._main_repo.odb.local.cache_dir
+            ),
+            self.PARAM_CACHE_TYPES: self._main_repo.odb.local.cache_types,
+            self.PARAM_SUBREPOS: self._traverse_subrepos,
+        }
+
+    @classmethod
+    def _repo_from_fs_config(
+        cls, **config
+    ) -> Tuple["Repo", Optional["RepoFactory"]]:
+        from dvc.external_repo import erepo_factory, external_repo
+        from dvc.repo import Repo
+
+        url = config.get(cls.PARAM_REPO_URL)
+        root = config.get(cls.PARAM_REPO_ROOT)
+        assert url or root
+
+        def _open(*args, **kwargs):
+            # NOTE: if original repo was an erepo (and has a URL),
+            # we cannot use Repo.open() since it will skip erepo
+            # cache/remote setup for local URLs
+            if url is None:
+                return Repo.open(*args, **kwargs)
+            return external_repo(*args, **kwargs)
+
+        cache_dir = config.get(cls.PARAM_CACHE_DIR)
+        cache_config = (
+            {}
+            if not cache_dir
+            else {
+                "cache": {
+                    "dir": cache_dir,
+                    "type": config.get(cls.PARAM_CACHE_TYPES),
+                }
+            }
+        )
+        repo_kwargs: dict = {
+            "rev": config.get(cls.PARAM_REV),
+            "subrepos": config.get(cls.PARAM_SUBREPOS, False),
+            "uninitialized": True,
+        }
+        factory: Optional["RepoFactory"] = None
+        if url is None:
+            repo_kwargs["config"] = cache_config
+        else:
+            repo_kwargs["cache_dir"] = cache_dir
+            factory = erepo_factory(url, cache_config)
+
+        with _open(
+            url if url else root,
+            **repo_kwargs,
+        ) as repo:
+            return repo, factory
+
+    def _get_repo(self, path: str) -> Optional["Repo"]:
         """Returns repo that the path falls in, using prefix.
 
         If the path is already tracked/collected, it just returns the repo.
@@ -93,11 +177,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             if self._is_dvc_repo(d):
                 repo = self.repo_factory(
                     d,
-                    scm=self.repo.scm,
-                    rev=self.repo.get_rev(),
+                    scm=self._main_repo.scm,
+                    rev=self._main_repo.get_rev(),
                     repo_factory=self.repo_factory,
                 )
-                self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
+                self._dvcfss[repo.root_dir] = DvcFileSystem(repo=repo)
             self._subrepos_trie[d] = repo
 
     def _is_dvc_repo(self, dir_path):
@@ -108,8 +192,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         from dvc.repo import Repo
 
         repo_path = os.path.join(dir_path, Repo.DVC_DIR)
-        # dvcignore will ignore subrepos, therefore using `use_dvcignore=False`
-        return self._main_repo.fs.isdir(repo_path, use_dvcignore=False)
+        return self._main_repo.fs.isdir(repo_path)
 
     def _get_fs_pair(
         self, path
@@ -128,7 +211,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
     def open(
         self, path, mode="r", encoding="utf-8", **kwargs
-    ):  # pylint: disable=arguments-differ
+    ):  # pylint: disable=arguments-renamed
         if "b" in mode:
             encoding = None
 
@@ -142,34 +225,38 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
 
         return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs)
 
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        fs, dvc_fs = self._get_fs_pair(path)
+    def exists(self, path_info) -> bool:
+        fs, dvc_fs = self._get_fs_pair(path_info)
 
         if not dvc_fs:
-            return fs.exists(path)
+            return fs.exists(path_info)
 
-        if fs.exists(path):
+        if dvc_fs.repo.dvcignore.is_ignored(fs, path_info):
+            return False
+
+        if fs.exists(path_info):
             return True
 
         try:
-            meta = dvc_fs.metadata(path)
+            meta = dvc_fs.metadata(path_info)
         except FileNotFoundError:
             return False
 
-        (out,) = meta.outs
-        assert len(meta.outs) == 1
-        if fs.exists(out.path_info):
-            return False
+        for out in meta.outs:
+            if fs.exists(out.path_info):
+                return False
+
         return True
 
-    def isdir(self, path):  # pylint: disable=arguments-differ
+    def isdir(self, path):  # pylint: disable=arguments-renamed
         fs, dvc_fs = self._get_fs_pair(path)
 
+        if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_dir(path):
+            return False
+
         try:
-            st = fs.stat(path)
-            return stat.S_ISDIR(st.st_mode)
+            info = fs.info(path)
+            return info["type"] == "directory"
         except (OSError, ValueError):
             # from CPython's os.path.isdir()
             pass
@@ -182,22 +269,25 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         except FileNotFoundError:
             return False
 
-        (out,) = meta.outs
-        assert len(meta.outs) == 1
-        if fs.exists(out.path_info):
-            return False
+        for out in meta.outs:
+            if fs.exists(out.path_info):
+                return False
+
         return meta.isdir
 
     def isdvc(self, path, **kwargs):
         _, dvc_fs = self._get_fs_pair(path)
         return dvc_fs is not None and dvc_fs.isdvc(path, **kwargs)
 
-    def isfile(self, path):  # pylint: disable=arguments-differ
+    def isfile(self, path):  # pylint: disable=arguments-renamed
         fs, dvc_fs = self._get_fs_pair(path)
 
+        if dvc_fs and dvc_fs.repo.dvcignore.is_ignored_file(path):
+            return False
+
         try:
-            st = fs.stat(path)
-            return stat.S_ISREG(st.st_mode)
+            info = fs.info(path)
+            return info["type"] == "file"
         except (OSError, ValueError):
             # from CPython's os.path.isfile()
             pass
@@ -222,10 +312,6 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return dvc_fs.isexec(path_info)
         return fs.isexec(path_info)
 
-    def stat(self, path):
-        fs, _ = self._get_fs_pair(path)
-        return fs.stat(path)
-
     def _dvc_walk(self, walk):
         try:
             root, dirs, files = next(walk)
@@ -238,13 +324,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     def _subrepo_walk(self, dir_path, **kwargs):
         """Walk into a new repo.
 
-         NOTE: subrepo will only be discovered when walking if
-         ignore_subrepos is set to False.
+        NOTE: subrepo will only be discovered when walking if
+        ignore_subrepos is set to False.
         """
         fs, dvc_fs = self._get_fs_pair(dir_path)
-        fs_walk = fs.walk(
-            dir_path, topdown=True, ignore_subrepos=not self._traverse_subrepos
-        )
+        fs_walk = fs.walk(dir_path, topdown=True)
         if dvc_fs:
             dvc_walk = dvc_fs.walk(dir_path, topdown=True, **kwargs)
         else:
@@ -306,15 +390,7 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             elif dirname in repo_set:
                 yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
 
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        dvcfiles=False,
-        follow_subrepos=None,
-        **kwargs
-    ):  # pylint: disable=arguments-differ
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
         """Walk and merge both DVC and repo fss.
 
         Args:
@@ -340,17 +416,14 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 onerror(NotADirectoryError(top))
             return
 
-        ignore_subrepos = not self._traverse_subrepos
-        if follow_subrepos is not None:
-            ignore_subrepos = not follow_subrepos
+        repo = self._get_repo(os.path.abspath(top))
+        dvcfiles = kwargs.pop("dvcfiles", False)
 
         fs, dvc_fs = self._get_fs_pair(top)
         repo_exists = fs.exists(top)
-        repo_walk = fs.walk(
-            top,
-            topdown=topdown,
-            onerror=onerror,
-            ignore_subrepos=ignore_subrepos,
+
+        repo_walk = repo.dvcignore.walk(
+            fs, top, topdown=topdown, onerror=onerror, **kwargs
         )
 
         if not dvc_fs or (repo_exists and dvc_fs.isdvc(top)):
@@ -358,36 +431,37 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return
 
         if not repo_exists:
-            yield from dvc_fs.walk(top, topdown=topdown, **kwargs)
+            yield from dvc_fs.walk(
+                top, topdown=topdown, onerror=onerror, **kwargs
+            )
 
         dvc_walk = None
         if dvc_fs.exists(top):
-            dvc_walk = dvc_fs.walk(top, topdown=topdown, **kwargs)
+            dvc_walk = dvc_fs.walk(
+                top, topdown=topdown, onerror=onerror, **kwargs
+            )
 
         yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
 
-    def walk_files(self, top, **kwargs):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top, **kwargs):
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info, **kwargs):
             for fname in files:
                 yield PathInfo(root) / fname
 
-    def _download(
-        self, from_info, to_file, name=None, no_progress_bar=False, **kwargs
-    ):
-        import shutil
+    def _download(self, from_info, to_file, **kwargs):
+        fs, dvc_fs = self._get_fs_pair(from_info)
+        try:
+            fs._download(  # pylint: disable=protected-access
+                from_info, to_file, **kwargs
+            )
+            return
+        except FileNotFoundError:
+            if not dvc_fs:
+                raise
 
-        from dvc.progress import Tqdm
-
-        with open(to_file, "wb+") as to_fobj:
-            with Tqdm.wrapattr(
-                to_fobj, "write", desc=name, disable=no_progress_bar,
-            ) as wrapped:
-                with self.open(from_info, "rb", **kwargs) as from_fobj:
-                    shutil.copyfileobj(from_fobj, wrapped)
-
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        return self._main_repo.fs.hash_jobs
+        dvc_fs._download(  # pylint: disable=protected-access
+            from_info, to_file, **kwargs
+        )
 
     def metadata(self, path):
         abspath = os.path.abspath(path)
@@ -399,11 +473,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             with suppress(FileNotFoundError):
                 dvc_meta = dvc_fs.metadata(path_info)
 
-        stat_result = None
+        info_result = None
         with suppress(FileNotFoundError):
-            stat_result = fs.stat(path_info)
+            info_result = fs.info(path_info)
 
-        if not stat_result and not dvc_meta:
+        if not info_result and not dvc_meta:
             raise FileNotFoundError
 
         from ._metadata import Metadata
@@ -413,13 +487,13 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             repo=self._get_repo(abspath) or self._main_repo,
         )
 
-        isdir = bool(stat_result) and stat.S_ISDIR(stat_result.st_mode)
+        isdir = bool(info_result) and info_result["type"] == "directory"
         meta.isdir = meta.isdir or isdir
 
         if not dvc_meta:
             from dvc.utils import is_exec
 
-            meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode)
+            meta.is_exec = bool(info_result) and is_exec(info_result["mode"])
         return meta
 
     def info(self, path_info):
@@ -429,3 +503,11 @@ class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             return fs.info(path_info)
         except FileNotFoundError:
             return dvc_fs.info(path_info)
+
+    def checksum(self, path_info):
+        fs, dvc_fs = self._get_fs_pair(path_info)
+
+        try:
+            return fs.checksum(path_info)
+        except FileNotFoundError:
+            return dvc_fs.checksum(path_info)

dvc/fs/s3.py

@@ -1,75 +1,34 @@
-import logging
+import functools
 import os
 import threading
-from contextlib import contextmanager
+from collections import defaultdict
 
 from funcy import cached_property, wrap_prop
 
-from dvc.config import ConfigError
-from dvc.exceptions import DvcException, ETagMismatchError
-from dvc.hash_info import HashInfo
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
-from dvc.utils import conversions, error_link
 
-from .base import BaseFileSystem
-
-logger = logging.getLogger(__name__)
+from .fsspec_wrapper import ObjectFSWrapper
 
 _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
 
 
-class S3FileSystem(BaseFileSystem):
+# pylint:disable=abstract-method
+class BaseS3FileSystem(ObjectFSWrapper):
     scheme = Schemes.S3
     PATH_CLS = CloudURLInfo
-    REQUIRES = {"boto3": "boto3"}
+    REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    _GRANTS = {
+        "grant_full_control": "GrantFullControl",
+        "grant_read": "GrantRead",
+        "grant_read_acp": "GrantReadACP",
+        "grant_write_acp": "GrantWriteACP",
+    }
 
-        url = config.get("url", "s3://")
-        self.path_info = self.PATH_CLS(url)
-
-        self.region = config.get("region")
-        self.profile = config.get("profile")
-        self.endpoint_url = config.get("endpointurl")
-
-        self.use_ssl = config.get("use_ssl", True)
-        self.ssl_verify = config.get("ssl_verify", True)
-
-        self.extra_args = {}
-
-        self.sse = config.get("sse")
-        if self.sse:
-            self.extra_args["ServerSideEncryption"] = self.sse
-
-        self.sse_kms_key_id = config.get("sse_kms_key_id")
-        if self.sse_kms_key_id:
-            self.extra_args["SSEKMSKeyId"] = self.sse_kms_key_id
-
-        self.acl = config.get("acl")
-        if self.acl:
-            self.extra_args["ACL"] = self.acl
-
-        self._append_aws_grants_to_extra_args(config)
-
-        self.access_key_id = config.get("access_key_id")
-        self.secret_access_key = config.get("secret_access_key")
-        self.session_token = config.get("session_token")
-
-        shared_creds = config.get("credentialpath")
-        if shared_creds:
-            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
-
-        config_path = config.get("configpath")
-        if config_path:
-            os.environ.setdefault("AWS_CONFIG_FILE", config_path)
-        self._transfer_config = None
-
-    # https://github.com/aws/aws-cli/blob/0376c6262d6b15dc36c82e6da6e1aad10249cc8c/awscli/customizations/s3/transferconfig.py#L107-L113
     _TRANSFER_CONFIG_ALIASES = {
         "max_queue_size": "max_io_queue",
         "max_concurrent_requests": "max_concurrency",
@@ -77,11 +36,15 @@ class S3FileSystem(BaseFileSystem):
         "multipart_chunksize": "multipart_chunksize",
     }
 
-    def _transform_config(self, s3_config):
+    def _split_s3_config(self, s3_config):
         """Splits the general s3 config into 2 different config
         objects, one for transfer.TransferConfig and other is the
         general session config"""
 
+        from boto3.s3.transfer import TransferConfig
+
+        from dvc.utils import conversions
+
         config, transfer_config = {}, {}
         for key, value in s3_config.items():
             if key in self._TRANSFER_CONFIG_ALIASES:
@@ -94,367 +57,183 @@ class S3FileSystem(BaseFileSystem):
             else:
                 config[key] = value
 
-        return config, transfer_config
+        # pylint: disable=attribute-defined-outside-init
+        self._transfer_config = TransferConfig(**transfer_config)
+        return config
 
-    def _process_config(self):
-        from boto3.s3.transfer import TransferConfig
+    def _load_aws_config_file(self, profile):
         from botocore.configloader import load_config
 
+        # pylint: disable=attribute-defined-outside-init
+        self._transfer_config = None
         config_path = os.environ.get("AWS_CONFIG_FILE", _AWS_CONFIG_PATH)
         if not os.path.exists(config_path):
-            return None
+            return {}
 
         config = load_config(config_path)
-        profile = config["profiles"].get(self.profile or "default")
-        if not profile:
-            return None
+        profile_config = config["profiles"].get(profile or "default")
+        if not profile_config:
+            return {}
 
-        s3_config = profile.get("s3", {})
-        s3_config, transfer_config = self._transform_config(s3_config)
-        self._transfer_config = TransferConfig(**transfer_config)
-        return s3_config
+        s3_config = profile_config.get("s3", {})
+        return self._split_s3_config(s3_config)
 
+    def _prepare_credentials(self, **config):
+        from dvc.config import ConfigError
+        from dvc.utils.flatten import flatten, unflatten
+
+        login_info = defaultdict(dict)
+
+        # credentials
+        login_info["key"] = config.get("access_key_id")
+        login_info["secret"] = config.get("secret_access_key")
+        login_info["token"] = config.get("session_token")
+
+        # session configuration
+        login_info["profile"] = config.get("profile")
+        login_info["use_ssl"] = config.get("use_ssl", True)
+
+        # extra client configuration
+        client = login_info["client_kwargs"]
+        client["region_name"] = config.get("region")
+        client["endpoint_url"] = config.get("endpointurl")
+        client["verify"] = config.get("ssl_verify")
+
+        # encryptions
+        additional = login_info["s3_additional_kwargs"]
+        additional["ServerSideEncryption"] = config.get("sse")
+        additional["SSEKMSKeyId"] = config.get("sse_kms_key_id")
+        additional["ACL"] = config.get("acl")
+        for grant_option, grant_key in self._GRANTS.items():
+            if config.get(grant_option):
+                if additional["ACL"]:
+                    raise ConfigError(
+                        "`acl` and `grant_*` AWS S3 config options "
+                        "are mutually exclusive"
+                    )
+                additional[grant_key] = config[grant_option]
+
+        # config kwargs
+        session_config = login_info["config_kwargs"]
+        session_config["s3"] = self._load_aws_config_file(
+            login_info["profile"]
+        )
+
+        shared_creds = config.get("credentialpath")
+        if shared_creds:
+            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
+
+        if (
+            client["region_name"] is None
+            and session_config["s3"].get("region_name") is None
+            and os.getenv("AWS_REGION") is None
+        ):
+            # Enable bucket region caching
+            login_info["cache_regions"] = config.get("cache_regions", True)
+
+        config_path = config.get("configpath")
+        if config_path:
+            os.environ.setdefault("AWS_CONFIG_FILE", config_path)
+
+        return unflatten(
+            {
+                key: value
+                for key, value in flatten(login_info).items()
+                if value is not None
+            }
+        )
+
+    def _entry_hook(self, entry):
+        entry = entry.copy()
+        if "ETag" in entry:
+            entry["etag"] = entry["ETag"].strip('"')
+        return entry
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def fs(self):
+        from s3fs import S3FileSystem as _S3FileSystem
+
+        return _S3FileSystem(**self.fs_args)
+
+
+def _translate_exceptions(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            func(*args, **kwargs)
+        except Exception as exc:
+            from s3fs.errors import translate_boto_error
+
+            raise translate_boto_error(exc)
+
+    return wrapper
+
+
+class S3FileSystem(BaseS3FileSystem):  # pylint:disable=abstract-method
     @wrap_prop(threading.Lock())
     @cached_property
     def s3(self):
         import boto3
 
+        login_info = self.fs_args
+        client_kwargs = login_info.get("client_kwargs", {})
         session_opts = {
-            "profile_name": self.profile,
-            "region_name": self.region,
+            "profile_name": login_info.get("profile"),
+            "region_name": client_kwargs.get("region_name"),
         }
 
-        if self.access_key_id:
-            session_opts["aws_access_key_id"] = self.access_key_id
-        if self.secret_access_key:
-            session_opts["aws_secret_access_key"] = self.secret_access_key
-        if self.session_token:
-            session_opts["aws_session_token"] = self.session_token
+        if "key" in login_info:
+            session_opts["aws_access_key_id"] = login_info["key"]
+        if "secret" in login_info:
+            session_opts["aws_secret_access_key"] = login_info["secret"]
+        if "token" in login_info:
+            session_opts["aws_session_token"] = login_info["token"]
 
         session = boto3.session.Session(**session_opts)
-        s3_config = self._process_config()
 
         return session.resource(
             "s3",
-            endpoint_url=self.endpoint_url,
-            verify=self.ssl_verify,
-            use_ssl=self.use_ssl,
-            config=boto3.session.Config(
-                signature_version="s3v4", s3=s3_config
-            ),
+            endpoint_url=client_kwargs.get("endpoint_url"),
+            use_ssl=login_info["use_ssl"],
+            verify=client_kwargs.get("verify"),
         )
 
-    @contextmanager
-    def _get_s3(self):
-        from botocore.exceptions import (
-            EndpointConnectionError,
-            NoCredentialsError,
-        )
-
-        try:
-            yield self.s3
-        except NoCredentialsError as exc:
-            link = error_link("no-credentials")
-            raise DvcException(
-                f"Unable to find AWS credentials. {link}"
-            ) from exc
-        except EndpointConnectionError as exc:
-            link = error_link("connection-error")
-            name = self.endpoint_url or "AWS S3"
-            raise DvcException(
-                f"Unable to connect to '{name}'. {link}"
-            ) from exc
-
-    @contextmanager
-    def _get_bucket(self, bucket):
-        with self._get_s3() as s3:
-            try:
-                yield s3.Bucket(bucket)
-            except s3.meta.client.exceptions.NoSuchBucket as exc:
-                link = error_link("no-bucket")
-                raise DvcException(
-                    f"Bucket '{bucket}' does not exist. {link}"
-                ) from exc
-
-    @contextmanager
     def _get_obj(self, path_info):
-        with self._get_bucket(path_info.bucket) as bucket:
-            try:
-                yield bucket.Object(path_info.path)
-            except bucket.meta.client.exceptions.NoSuchKey as exc:
-                raise DvcException(f"{path_info.url} does not exist") from exc
-
-    def _append_aws_grants_to_extra_args(self, config):
-        # Keys for extra_args can be one of the following list:
-        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
-        """
-          ALLOWED_UPLOAD_ARGS = [
-            'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
-            'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
-            'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
-            'RequestPayer', 'ServerSideEncryption', 'StorageClass',
-            'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
-            'SSEKMSKeyId', 'WebsiteRedirectLocation'
-          ]
-        """
-
-        grants = {
-            "grant_full_control": "GrantFullControl",
-            "grant_read": "GrantRead",
-            "grant_read_acp": "GrantReadACP",
-            "grant_write_acp": "GrantWriteACP",
-        }
-
-        for grant_option, extra_args_key in grants.items():
-            if config.get(grant_option):
-                if self.acl:
-                    raise ConfigError(
-                        "`acl` and `grant_*` AWS S3 config options "
-                        "are mutually exclusive"
-                    )
-
-                self.extra_args[extra_args_key] = config.get(grant_option)
-
-    def _generate_download_url(self, path_info, expires=3600):
-        params = {"Bucket": path_info.bucket, "Key": path_info.path}
-        with self._get_s3() as s3:
-            return s3.meta.client.generate_presigned_url(
-                ClientMethod="get_object",
-                Params=params,
-                ExpiresIn=int(expires),
-            )
-
-    def exists(self, path_info, use_dvcignore=True):
-        """Check if the blob exists. If it does not exist,
-        it could be a part of a directory path.
-
-        eg: if `data/file.txt` exists, check for `data` should return True
-        """
-        return self.isfile(path_info) or self.isdir(path_info)
-
-    def isdir(self, path_info):
-        # S3 doesn't have a concept for directories.
-        #
-        # Using `head_object` with a path pointing to a directory
-        # will throw a 404 error.
-        #
-        # A reliable way to know if a given path is a directory is by
-        # checking if there are more files sharing the same prefix
-        # with a `list_objects` call.
-        #
-        # We need to make sure that the path ends with a forward slash,
-        # since we can end with false-positives like the following example:
-        #
-        #       bucket
-        #       └── data
-        #          ├── alice
-        #          └── alpha
-        #
-        # Using `data/al` as prefix will return `[data/alice, data/alpha]`,
-        # While `data/al/` will return nothing.
-        #
-        dir_path = path_info / ""
-        return bool(list(self._list_paths(dir_path, max_items=1)))
-
-    def isfile(self, path_info):
-        if path_info.path.endswith("/"):
-            return False
-
-        return path_info.path in self._list_paths(path_info)
-
-    def info(self, path_info):
-        # temporary workaround, will be removed when migrating to s3fs
-        if self.isdir(path_info):
-            return {"type": "directory"}
-
-        with self._get_obj(path_info) as obj:
-            return {
-                "type": "file",
-                "etag": obj.e_tag.strip('"'),
-                "size": obj.content_length,
-            }
-
-    def _list_paths(self, path_info, max_items=None):
-        with self._get_bucket(path_info.bucket) as bucket:
-            obj_summaries = bucket.objects.filter(Prefix=path_info.path)
-            if max_items is not None:
-                obj_summaries = obj_summaries.page_size(max_items).limit(
-                    max_items
-                )
-            for obj_summary in obj_summaries:
-                yield obj_summary.key
-
-    def walk_files(self, path_info, **kwargs):
-        if not kwargs.pop("prefix", False):
-            path_info = path_info / ""
-        for fname in self._list_paths(path_info, **kwargs):
-            if fname.endswith("/"):
-                continue
-
-            yield path_info.replace(path=fname)
-
-    def find(self, path_info, detail=False):
-        with self._get_bucket(path_info.bucket) as bucket:
-            for obj_summary in bucket.objects.filter(Prefix=path_info.path):
-                if detail:
-                    yield {
-                        "type": "file",
-                        "name": obj_summary.key,
-                        "size": obj_summary.size,
-                        "etag": obj_summary.e_tag.strip('"'),
-                    }
-                else:
-                    yield obj_summary.key
-
-    def remove(self, path_info):
-        if path_info.scheme != "s3":
-            raise NotImplementedError
-
-        logger.debug(f"Removing {path_info}")
-        with self._get_obj(path_info) as obj:
-            obj.delete()
-
-    def makedirs(self, path_info):
-        # We need to support creating empty directories, which means
-        # creating an object with an empty body and a trailing slash `/`.
-        #
-        # We are not creating directory objects for every parent prefix,
-        # as it is not required.
-        if not path_info.path:
-            return
-
-        dir_path = path_info / ""
-        with self._get_obj(dir_path) as obj:
-            obj.put(Body="")
-
-    def copy(self, from_info, to_info):
-        with self._get_s3() as s3:
-            self._copy(s3.meta.client, from_info, to_info, self.extra_args)
-
-    @classmethod
-    def _copy_multipart(
-        cls, s3, from_info, to_info, size, n_parts, extra_args
-    ):
-        mpu = s3.create_multipart_upload(
-            Bucket=to_info.bucket, Key=to_info.path, **extra_args
-        )
-        mpu_id = mpu["UploadId"]
-
-        parts = []
-        byte_position = 0
-        for i in range(1, n_parts + 1):
-            obj = s3.head_object(
-                Bucket=from_info.bucket, Key=from_info.path, PartNumber=i
-            )
-            part_size = obj["ContentLength"]
-            lastbyte = byte_position + part_size - 1
-            if lastbyte > size:
-                lastbyte = size - 1
-
-            srange = f"bytes={byte_position}-{lastbyte}"
-
-            part = s3.upload_part_copy(
-                Bucket=to_info.bucket,
-                Key=to_info.path,
-                PartNumber=i,
-                UploadId=mpu_id,
-                CopySourceRange=srange,
-                CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
-            )
-            parts.append(
-                {"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
-            )
-            byte_position += part_size
-
-        assert n_parts == len(parts)
-
-        s3.complete_multipart_upload(
-            Bucket=to_info.bucket,
-            Key=to_info.path,
-            UploadId=mpu_id,
-            MultipartUpload={"Parts": parts},
-        )
-
-    @classmethod
-    def _copy(cls, s3, from_info, to_info, extra_args):
-        # NOTE: object's etag depends on the way it was uploaded to s3 or the
-        # way it was copied within the s3. More specifically, it depends on
-        # the chunk size that was used to transfer it, which would affect
-        # whether an object would be uploaded as a single part or as a
-        # multipart.
-        #
-        # If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
-        # then it was transferred as a single part, which means that the chunk
-        # size used to transfer it was greater or equal to the ContentLength
-        # of that object. So to preserve that tag over the next transfer, we
-        # could use any value >= ContentLength.
-        #
-        # If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
-        # then it was transferred as a multipart, which means that the chunk
-        # size used to transfer it was less than ContentLength of that object.
-        # Unfortunately, in general, it doesn't mean that the chunk size was
-        # the same throughout the transfer, so it means that in order to
-        # preserve etag, we need to transfer each part separately, so the
-        # object is transfered in the same chunks as it was originally.
-        from boto3.s3.transfer import TransferConfig
-
-        obj = s3.head_object(Bucket=from_info.bucket, Key=from_info.path)
-        etag = obj["ETag"].strip('"')
-        size = obj["ContentLength"]
-
-        _, _, parts_suffix = etag.partition("-")
-        if parts_suffix:
-            n_parts = int(parts_suffix)
-            cls._copy_multipart(
-                s3, from_info, to_info, size, n_parts, extra_args=extra_args
-            )
-        else:
-            source = {"Bucket": from_info.bucket, "Key": from_info.path}
-            s3.copy(
-                source,
-                to_info.bucket,
-                to_info.path,
-                ExtraArgs=extra_args,
-                Config=TransferConfig(multipart_threshold=size + 1),
-            )
-
-        cached_etag = s3.head_object(Bucket=to_info.bucket, Key=to_info.path)[
-            "ETag"
-        ].strip('"')
-        if etag != cached_etag:
-            raise ETagMismatchError(etag, cached_etag)
-
-    def etag(self, path_info):
-        with self._get_obj(path_info) as obj:
-            return HashInfo("etag", size=obj.content_length,)
-
-    def _upload_fobj(self, fobj, to_info):
-        with self._get_obj(to_info) as obj:
-            obj.upload_fileobj(fobj, Config=self._transfer_config)
+        bucket = self.s3.Bucket(path_info.bucket)
+        return bucket.Object(path_info.path)
 
+    @_translate_exceptions
     def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+        self, from_file, to_info, name=None, no_progress_bar=False, **pbar_args
     ):
-        with self._get_obj(to_info) as obj:
-            total = os.path.getsize(from_file)
-            with Tqdm(
-                disable=no_progress_bar, total=total, bytes=True, desc=name
-            ) as pbar:
-                obj.upload_file(
-                    from_file,
-                    Callback=pbar.update,
-                    ExtraArgs=self.extra_args,
-                    Config=self._transfer_config,
-                )
+        total = os.path.getsize(from_file)
+        with Tqdm(
+            disable=no_progress_bar,
+            total=total,
+            bytes=True,
+            desc=name,
+            **pbar_args,
+        ) as pbar:
+            obj = self._get_obj(to_info)
+            obj.upload_file(
+                from_file,
+                Callback=pbar.update,
+                ExtraArgs=self.fs_args.get("s3_additional_kwargs"),
+                Config=self._transfer_config,
+            )
+        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
 
-    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        with self._get_obj(from_info) as obj:
-            with Tqdm(
-                disable=no_progress_bar,
-                total=obj.content_length,
-                bytes=True,
-                desc=name,
-            ) as pbar:
-                obj.download_file(
-                    to_file, Callback=pbar.update, Config=self._transfer_config
-                )
+    @_translate_exceptions
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
+    ):
+        obj = self._get_obj(from_info)
+        with Tqdm(
+            disable=no_progress_bar,
+            total=obj.content_length,
+            bytes=True,
+            desc=name,
+            **pbar_args,
+        ) as pbar:
+            obj.download_file(to_file, Callback=pbar.update)

dvc/fs/ssh.py

@@ -0,0 +1,128 @@
+import getpass
+import os.path
+import threading
+
+from funcy import cached_property, memoize, silent, wrap_prop, wrap_with
+
+from dvc import prompt
+from dvc.scheme import Schemes
+from dvc.utils.fs import as_atomic
+
+from .fsspec_wrapper import CallbackMixin, FSSpecWrapper
+
+_SSH_TIMEOUT = 60 * 30
+_SSH_CONFIG_FILE = os.path.expanduser(os.path.join("~", ".ssh", "config"))
+
+
+@wrap_with(threading.Lock())
+@memoize
+def ask_password(host, user, port):
+    return prompt.password(
+        "Enter a private key passphrase or a password for "
+        "host '{host}' port '{port}' user '{user}'".format(
+            host=host, port=port, user=user
+        )
+    )
+
+
+# pylint:disable=abstract-method
+class SSHFileSystem(CallbackMixin, FSSpecWrapper):
+    scheme = Schemes.SSH
+    REQUIRES = {"sshfs": "sshfs"}
+
+    DEFAULT_PORT = 22
+    PARAM_CHECKSUM = "md5"
+
+    def _with_bucket(self, path):
+        if isinstance(path, self.PATH_CLS):
+            path_str = path.path
+        else:
+            path_str = super()._with_bucket(path)
+        return os.path.normpath(path_str)
+
+    def _prepare_credentials(self, **config):
+        self.CAN_TRAVERSE = True
+        from sshfs.config import parse_config
+
+        login_info = {}
+
+        try:
+            user_ssh_config = parse_config(host=config["host"])
+        except FileNotFoundError:
+            user_ssh_config = {}
+
+        login_info["host"] = user_ssh_config.get("Hostname", config["host"])
+
+        login_info["username"] = (
+            config.get("user")
+            or config.get("username")
+            or user_ssh_config.get("User")
+            or getpass.getuser()
+        )
+
+        login_info["port"] = (
+            config.get("port")
+            or silent(int)(user_ssh_config.get("Port"))
+            or self.DEFAULT_PORT
+        )
+
+        login_info["password"] = config.get("password")
+
+        raw_keys = []
+        if config.get("keyfile"):
+            raw_keys.append(config.get("keyfile"))
+        elif user_ssh_config.get("IdentityFile"):
+            raw_keys.extend(user_ssh_config.get("IdentityFile"))
+
+        if raw_keys:
+            login_info["client_keys"] = [
+                os.path.expanduser(key) for key in raw_keys
+            ]
+
+        login_info["timeout"] = config.get("timeout", _SSH_TIMEOUT)
+
+        # These two settings fine tune the asyncssh to use the
+        # fastest encryption algorithm and disable compression
+        # altogether (since it blocking, it is slowing down
+        # the transfers in a considerable rate, and even for
+        # compressible data it is making it extremely slow).
+        # See: https://github.com/ronf/asyncssh/issues/374
+        login_info["encryption_algs"] = [
+            "[email protected]",
+            "aes256-ctr",
+            "aes192-ctr",
+            "aes128-ctr",
+        ]
+        login_info["compression_algs"] = None
+
+        login_info["gss_auth"] = config.get("gss_auth", False)
+        login_info["agent_forwarding"] = config.get("agent_forwarding", True)
+        login_info["proxy_command"] = user_ssh_config.get("ProxyCommand")
+
+        if config.get("ask_password") and login_info["password"] is None:
+            login_info["password"] = ask_password(
+                login_info["host"], login_info["username"], login_info["port"]
+            )
+
+        # We are going to automatically add stuff to known_hosts
+        # something like paramiko's AutoAddPolicy()
+        login_info["known_hosts"] = None
+        return login_info
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def fs(self):
+        from sshfs import SSHFileSystem as _SSHFileSystem
+
+        return _SSHFileSystem(**self.fs_args)
+
+    # Ensure that if an interrupt happens during the transfer, we don't
+    # pollute the cache.
+
+    def _upload_fobj(self, fobj, to_info, *args, **kwargs):
+        with as_atomic(self, to_info) as tmp_file:
+            super()._upload_fobj(fobj, tmp_file, *args, **kwargs)
+
+    def _upload(self, from_file, to_info, *args, **kwargs):
+        with as_atomic(self, to_info) as tmp_file:
+            super()._upload(from_file, tmp_file, *args, **kwargs)

dvc/fs/ssh/__init__.py

@@ -1,273 +0,0 @@
-import getpass
-import io
-import logging
-import os
-import shutil
-import threading
-from contextlib import closing, contextmanager
-from urllib.parse import urlparse
-
-from funcy import first, memoize, silent, wrap_with
-
-import dvc.prompt as prompt
-from dvc.hash_info import HashInfo
-from dvc.scheme import Schemes
-
-from ..base import BaseFileSystem
-from ..pool import get_connection
-
-logger = logging.getLogger(__name__)
-
-
-@wrap_with(threading.Lock())
-@memoize
-def ask_password(host, user, port):
-    return prompt.password(
-        "Enter a private key passphrase or a password for "
-        "host '{host}' port '{port}' user '{user}'".format(
-            host=host, port=port, user=user
-        )
-    )
-
-
-class SSHFileSystem(BaseFileSystem):
-    scheme = Schemes.SSH
-    REQUIRES = {"paramiko": "paramiko"}
-    _JOBS = 4
-
-    PARAM_CHECKSUM = "md5"
-    # At any given time some of the connections will go over network and
-    # paramiko stuff, so we would ideally have it double of server processors.
-    # We use conservative setting of 4 instead to not exhaust max sessions.
-    CHECKSUM_JOBS = 4
-    DEFAULT_CACHE_TYPES = ["copy"]
-    TRAVERSE_PREFIX_LEN = 2
-
-    DEFAULT_PORT = 22
-    TIMEOUT = 1800
-
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-        url = config.get("url")
-        if url:
-            parsed = urlparse(url)
-            user_ssh_config = self._load_user_ssh_config(parsed.hostname)
-
-            host = user_ssh_config.get("hostname", parsed.hostname)
-            user = (
-                config.get("user")
-                or parsed.username
-                or user_ssh_config.get("user")
-                or getpass.getuser()
-            )
-            port = (
-                config.get("port")
-                or parsed.port
-                or self._try_get_ssh_config_port(user_ssh_config)
-                or self.DEFAULT_PORT
-            )
-            self.path_info = self.PATH_CLS.from_parts(
-                scheme=self.scheme,
-                host=host,
-                user=user,
-                port=port,
-                path=parsed.path,
-            )
-        else:
-            self.path_info = None
-            user_ssh_config = {}
-
-        self.keyfile = config.get(
-            "keyfile"
-        ) or self._try_get_ssh_config_keyfile(user_ssh_config)
-        self.timeout = config.get("timeout", self.TIMEOUT)
-        self.password = config.get("password", None)
-        self.ask_password = config.get("ask_password", False)
-        self.gss_auth = config.get("gss_auth", False)
-        proxy_command = user_ssh_config.get("proxycommand", False)
-        if proxy_command:
-            import paramiko
-
-            self.sock = paramiko.ProxyCommand(proxy_command)
-        else:
-            self.sock = None
-        self.allow_agent = config.get("allow_agent", True)
-
-    @staticmethod
-    def ssh_config_filename():
-        return os.path.expanduser(os.path.join("~", ".ssh", "config"))
-
-    @staticmethod
-    def _load_user_ssh_config(hostname):
-        import paramiko
-
-        user_config_file = SSHFileSystem.ssh_config_filename()
-        user_ssh_config = {}
-        if hostname and os.path.exists(user_config_file):
-            ssh_config = paramiko.SSHConfig()
-            with open(user_config_file) as f:
-                # For whatever reason parsing directly from f is unreliable
-                f_copy = io.StringIO(f.read())
-                ssh_config.parse(f_copy)
-            user_ssh_config = ssh_config.lookup(hostname)
-        return user_ssh_config
-
-    @staticmethod
-    def _try_get_ssh_config_port(user_ssh_config):
-        return silent(int)(user_ssh_config.get("port"))
-
-    @staticmethod
-    def _try_get_ssh_config_keyfile(user_ssh_config):
-        return first(user_ssh_config.get("identityfile") or ())
-
-    def ensure_credentials(self, path_info=None):
-        if path_info is None:
-            path_info = self.path_info
-
-        # NOTE: we use the same password regardless of the server :(
-        if self.ask_password and self.password is None:
-            host, user, port = path_info.host, path_info.user, path_info.port
-            self.password = ask_password(host, user, port)
-
-    def ssh(self, path_info):
-        self.ensure_credentials(path_info)
-
-        from .connection import SSHConnection
-
-        return get_connection(
-            SSHConnection,
-            path_info.host,
-            username=path_info.user,
-            port=path_info.port,
-            key_filename=self.keyfile,
-            timeout=self.timeout,
-            password=self.password,
-            gss_auth=self.gss_auth,
-            sock=self.sock,
-            allow_agent=self.allow_agent,
-        )
-
-    @contextmanager
-    def open(self, path_info, mode="r", encoding=None, **kwargs):
-        assert mode in {"r", "rt", "rb", "wb"}
-
-        with self.ssh(path_info) as ssh, closing(
-            ssh.sftp.open(path_info.path, mode)
-        ) as fd:
-            if "b" in mode:
-                yield fd
-            else:
-                yield io.TextIOWrapper(fd, encoding=encoding)
-
-    def exists(self, path_info, use_dvcignore=True):
-        with self.ssh(path_info) as ssh:
-            return ssh.exists(path_info.path)
-
-    def isdir(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.isdir(path_info.path)
-
-    def isfile(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.isfile(path_info.path)
-
-    def walk_files(self, path_info, **kwargs):
-        with self.ssh(path_info) as ssh:
-            for fname in ssh.walk_files(path_info.path):
-                yield path_info.replace(path=fname)
-
-    def remove(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(path_info) as ssh:
-            ssh.remove(path_info.path)
-
-    def makedirs(self, path_info):
-        with self.ssh(path_info) as ssh:
-            ssh.makedirs(path_info.path)
-
-    def move(self, from_info, to_info):
-        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.move(from_info.path, to_info.path)
-
-    def copy(self, from_info, to_info):
-        if not from_info.scheme == to_info.scheme == self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.atomic_copy(from_info.path, to_info.path)
-
-    def symlink(self, from_info, to_info):
-        if not from_info.scheme == to_info.scheme == self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.symlink(from_info.path, to_info.path)
-
-    def hardlink(self, from_info, to_info):
-        if not from_info.scheme == to_info.scheme == self.scheme:
-            raise NotImplementedError
-
-        # See dvc/remote/local/__init__.py - hardlink()
-        if self.getsize(from_info) == 0:
-
-            with self.ssh(to_info) as ssh:
-                ssh.sftp.open(to_info.path, "w").close()
-
-            logger.debug(
-                "Created empty file: {src} -> {dest}".format(
-                    src=str(from_info), dest=str(to_info)
-                )
-            )
-            return
-
-        with self.ssh(from_info) as ssh:
-            ssh.hardlink(from_info.path, to_info.path)
-
-    def reflink(self, from_info, to_info):
-        if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        with self.ssh(from_info) as ssh:
-            ssh.reflink(from_info.path, to_info.path)
-
-    def md5(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return HashInfo(
-                "md5",
-                ssh.md5(path_info.path),
-                size=ssh.getsize(path_info.path),
-            )
-
-    def info(self, path_info):
-        with self.ssh(path_info) as ssh:
-            return ssh.info(path_info.path)
-
-    def _upload_fobj(self, fobj, to_info):
-        self.makedirs(to_info.parent)
-        with self.open(to_info, mode="wb") as fdest:
-            shutil.copyfileobj(fobj, fdest)
-
-    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        with self.ssh(from_info) as ssh:
-            ssh.download(
-                from_info.path,
-                to_file,
-                progress_title=name,
-                no_progress_bar=no_progress_bar,
-            )
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
-    ):
-        with self.ssh(to_info) as ssh:
-            ssh.upload(
-                from_file,
-                to_info.path,
-                progress_title=name,
-                no_progress_bar=no_progress_bar,
-            )

dvc/fs/ssh/connection.py

@@ -1,356 +0,0 @@
-import errno
-import logging
-import os
-import posixpath
-import shlex
-import stat
-from contextlib import suppress
-
-from funcy import cached_property
-
-from dvc.exceptions import DvcException
-from dvc.progress import Tqdm
-from dvc.utils import tmp_fname
-
-from ..base import RemoteCmdError
-
-try:
-    import paramiko
-except ImportError:
-    paramiko = None  # type: ignore
-
-
-logger = logging.getLogger(__name__)
-
-
-def sizeof_fmt(num, suffix="B"):
-    """ Convert number of bytes to human-readable string """
-    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
-        if abs(num) < 1024.0:
-            return f"{num:3.1f}{unit}{suffix}"
-        num /= 1024.0
-    return "{:.1f}{}{}".format(num, "Y", suffix)
-
-
-class SSHConnection:
-    def __init__(self, host, *args, **kwargs):
-        logger.debug(
-            "Establishing ssh connection with '{host}' "
-            "through port '{port}' as user '{username}'".format(
-                host=host, **kwargs
-            )
-        )
-
-        kwargs.setdefault("timeout", 1800)
-        self.timeout = kwargs["timeout"]
-
-        self._ssh = paramiko.SSHClient()
-
-        # Explicitly disable paramiko logger. Due to how paramiko dynamically
-        # loads loggers, it is not disabled by DVC disable_other_loggers().
-        # See https://github.com/iterative/dvc/issues/3482
-        self._ssh.set_log_channel("dvc.paramiko")
-        logging.getLogger("dvc.paramiko").disabled = True
-        logging.getLogger("dvc.paramiko.sftp").disabled = True
-
-        self._ssh.load_system_host_keys()
-        self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        self._ssh.connect(host, *args, **kwargs)
-        self._ssh.get_transport().set_keepalive(10)
-        self._sftp_channels = []
-
-    @property
-    def sftp(self):
-        if not self._sftp_channels:
-            self._sftp_channels = [self._ssh.open_sftp()]
-        return self._sftp_channels[0]
-
-    def close(self):
-        for sftp in self._sftp_channels:
-            sftp.close()
-        self._ssh.close()
-
-    def st_mode(self, path):
-        lstat = None
-        with suppress(FileNotFoundError):
-            lstat = self.sftp.lstat(path)
-
-        return lstat.st_mode if lstat else 0
-
-    def info(self, path):
-        st = self.sftp.stat(path)
-        return {
-            "size": st.st_size,
-            "type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
-        }
-
-    def getsize(self, path):
-        return self.info(path)["size"]
-
-    def exists(self, path):
-        return bool(self.st_mode(path))
-
-    def isdir(self, path):
-        return stat.S_ISDIR(self.st_mode(path))
-
-    def isfile(self, path):
-        return stat.S_ISREG(self.st_mode(path))
-
-    def islink(self, path):
-        return stat.S_ISLNK(self.st_mode(path))
-
-    def makedirs(self, path):
-        # Single stat call will say whether this is a dir, a file or a link
-        st_mode = self.st_mode(path)
-
-        if stat.S_ISDIR(st_mode):
-            return
-
-        if stat.S_ISREG(st_mode) or stat.S_ISLNK(st_mode):
-            raise DvcException(
-                f"a file with the same name '{path}' already exists"
-            )
-
-        head, tail = posixpath.split(path)
-
-        if head:
-            self.makedirs(head)
-
-        if tail:
-            try:
-                self.sftp.mkdir(path)
-            except OSError as exc:
-                # Since paramiko errors are very vague we need to recheck
-                # whether it's because path already exists or something else
-                if exc.errno == errno.EACCES or not self.exists(path):
-                    raise DvcException(
-                        f"unable to create remote directory '{path}'"
-                    ) from exc
-
-    def walk(self, directory, topdown=True):
-        # NOTE: original os.walk() implementation [1] with default options was
-        # used as a template.
-        #
-        # [1] https://github.com/python/cpython/blob/master/Lib/os.py
-        try:
-            dir_entries = self.sftp.listdir_attr(directory)
-        except OSError as exc:
-            raise DvcException(
-                "couldn't get the '{}' remote directory files list".format(
-                    directory
-                )
-            ) from exc
-
-        dirs = []
-        nondirs = []
-        for entry in dir_entries:
-            name = entry.filename
-            if stat.S_ISDIR(entry.st_mode):
-                dirs.append(name)
-            else:
-                nondirs.append(name)
-
-        if topdown:
-            yield directory, dirs, nondirs
-
-        for dname in dirs:
-            newpath = posixpath.join(directory, dname)
-            yield from self.walk(newpath, topdown=topdown)
-
-        if not topdown:
-            yield directory, dirs, nondirs
-
-    def walk_files(self, directory):
-        for root, _, files in self.walk(directory):
-            for fname in files:
-                yield posixpath.join(root, fname)
-
-    def _remove_file(self, path):
-        with suppress(FileNotFoundError):
-            self.sftp.remove(path)
-
-    def _remove_dir(self, path):
-        for root, dirs, files in self.walk(path, topdown=False):
-            for fname in files:
-                with suppress(FileNotFoundError):
-                    self._remove_file(posixpath.join(root, fname))
-
-            for dname in dirs:
-                with suppress(FileNotFoundError):
-                    self.sftp.rmdir(posixpath.join(root, dname))
-
-        with suppress(FileNotFoundError):
-            self.sftp.rmdir(path)
-
-    def remove(self, path):
-        if self.isdir(path):
-            self._remove_dir(path)
-        else:
-            self._remove_file(path)
-
-    def download(self, src, dest, no_progress_bar=False, progress_title=None):
-        with Tqdm(
-            desc=progress_title or os.path.basename(src),
-            disable=no_progress_bar,
-            bytes=True,
-            total=self.getsize(src),
-        ) as pbar:
-            self.sftp.get(src, dest, callback=pbar.update_to)
-
-    def move(self, src, dst):
-        """Rename src to dst, if it is not possible (in case src and dst are
-        on different filesystems) and actual physical copying of data is
-        happening.
-        """
-        self.makedirs(posixpath.dirname(dst))
-
-        try:
-            self.sftp.rename(src, dst)
-        except OSError:
-            self.atomic_copy(src, dst)
-            self.remove(src)
-
-    def atomic_copy(self, src, dst):
-        tmp = tmp_fname(dst)
-
-        try:
-            self.copy(src, tmp)
-            self.sftp.rename(tmp, dst)
-        finally:
-            self.remove(tmp)
-
-    def upload(self, src, dest, no_progress_bar=False, progress_title=None):
-        self.makedirs(posixpath.dirname(dest))
-        tmp_file = tmp_fname(dest)
-        if not progress_title:
-            progress_title = posixpath.basename(dest)
-
-        with Tqdm(
-            desc=progress_title,
-            disable=no_progress_bar,
-            bytes=True,
-            total=os.path.getsize(src),
-        ) as pbar:
-            self.sftp.put(src, tmp_file, callback=pbar.update_to)
-
-        self.sftp.rename(tmp_file, dest)
-
-    def execute(self, cmd):
-        stdin, stdout, stderr = self._ssh.exec_command(cmd)
-        channel = stdout.channel
-
-        stdin.close()
-        channel.shutdown_write()
-
-        stdout_chunks = []
-        stderr_chunks = []
-        while (
-            not channel.closed
-            or channel.recv_ready()
-            or channel.recv_stderr_ready()
-        ):
-            import selectors
-
-            selector = selectors.DefaultSelector()
-            selector.register(stdout.channel, selectors.EVENT_READ)
-
-            got_chunk = False
-            events = selector.select(self.timeout)
-            for key, _ in events:
-                c = key.fileobj
-                if c.recv_ready():
-                    stdout_chunks.append(stdout.channel.recv(len(c.in_buffer)))
-                    got_chunk = True
-
-                if c.recv_stderr_ready():
-                    stderr_len = len(c.in_stderr_buffer)
-                    s = stderr.channel.recv_stderr(stderr_len)
-                    stderr_chunks.append(s)
-                    got_chunk = True
-
-            if (
-                not got_chunk
-                and stdout.channel.exit_status_ready()
-                and not stderr.channel.recv_stderr_ready()
-                and not stdout.channel.recv_ready()
-            ):
-                stdout.channel.shutdown_read()
-                stdout.channel.close()
-                break
-
-        stdout.close()
-        stderr.close()
-
-        ret = stdout.channel.recv_exit_status()
-        if ret != 0:
-            err = b"".join(stderr_chunks).decode("utf-8")
-            raise RemoteCmdError("ssh", cmd, ret, err)
-
-        return b"".join(stdout_chunks).decode("utf-8")
-
-    @cached_property
-    def uname(self):
-        return self.execute("uname").strip()
-
-    def md5(self, path):
-        """
-        Use different md5 commands depending on the OS:
-
-         - Darwin's `md5` returns BSD-style checksums by default
-         - Linux's `md5sum` needs the `--tag` flag for a similar output
-
-         Example:
-              MD5 (foo.txt) = f3d220a856b52aabbf294351e8a24300
-        """
-        path = shlex.quote(path)
-        if self.uname == "Linux":
-            md5 = self.execute("md5sum " + path).split()[0]
-        elif self.uname == "Darwin":
-            md5 = self.execute("md5 " + path).split()[-1]
-        else:
-            raise DvcException(
-                f"'{self.uname}' is not supported as a SSH remote"
-            )
-
-        assert len(md5) == 32
-        return md5
-
-    def copy(self, src, dest):
-        dest = shlex.quote(dest)
-        src = shlex.quote(src)
-        self.execute(f"cp {src} {dest}")
-
-    def open_max_sftp_channels(self):
-        # If there are more than 1 it means we've already opened max amount
-        if len(self._sftp_channels) <= 1:
-            while True:
-                try:
-                    self._sftp_channels.append(self._ssh.open_sftp())
-                except paramiko.ssh_exception.ChannelException:
-                    if not self._sftp_channels:
-                        raise
-                    break
-        return self._sftp_channels
-
-    def open(self, *args, **kwargs):
-        return self.sftp.open(*args, **kwargs)
-
-    def symlink(self, src, dest):
-        self.sftp.symlink(src, dest)
-
-    def reflink(self, src, dest):
-        dest = shlex.quote(dest)
-        src = shlex.quote(src)
-        if self.uname == "Linux":
-            return self.execute(f"cp --reflink {src} {dest}")
-
-        if self.uname == "Darwin":
-            return self.execute(f"cp -c {src} {dest}")
-
-        raise DvcException(f"'{self.uname}' is not supported as a SSH remote")
-
-    def hardlink(self, src, dest):
-        dest = shlex.quote(dest)
-        src = shlex.quote(src)
-        self.execute(f"ln {src} {dest}")

dvc/fs/utils.py

@@ -0,0 +1,55 @@
+import logging
+import os
+from typing import TYPE_CHECKING
+
+from .local import LocalFileSystem
+
+if TYPE_CHECKING:
+    from dvc.types import AnyPath, DvcPath
+
+    from .base import BaseFileSystem
+
+logger = logging.getLogger(__name__)
+
+
+def transfer(
+    from_fs: "BaseFileSystem",
+    from_info: "AnyPath",
+    to_fs: "BaseFileSystem",
+    to_info: "DvcPath",
+    move: bool = False,
+) -> None:
+    use_move = isinstance(from_fs, type(to_fs)) and move
+    try:
+        if use_move:
+            return to_fs.move(from_info, to_info)
+
+        if isinstance(from_fs, LocalFileSystem):
+            if not isinstance(from_info, from_fs.PATH_CLS):
+                from_info = from_fs.PATH_CLS(from_info)
+            return to_fs.upload(from_info, to_info)
+
+        if isinstance(to_fs, LocalFileSystem):
+            return from_fs.download_file(from_info, to_info)
+
+        with from_fs.open(from_info, mode="rb") as fobj:
+            return to_fs.upload_fobj(fobj, to_info)
+    except OSError as exc:
+        # If the target file already exists, we are going to simply
+        # ignore the exception (#4992).
+        #
+        # On Windows, it is not always guaranteed that you'll get
+        # FileExistsError (it might be PermissionError or a bare OSError)
+        # but all of those exceptions raised from the original
+        # FileExistsError so we have a separate check for that.
+        if isinstance(exc, FileExistsError) or (
+            os.name == "nt"
+            and exc.__context__
+            and isinstance(exc.__context__, FileExistsError)
+        ):
+            logger.debug("'%s' file already exists, skipping", to_info)
+            if use_move:
+                from_fs.remove(from_info)
+            return None
+
+        raise

dvc/fs/webdav.py

@@ -1,273 +1,94 @@
-import io
 import logging
 import os
 import threading
-from collections import deque
+from functools import lru_cache
 
-from funcy import cached_property, nullcontext, wrap_prop
+from funcy import cached_property, wrap_prop
 
-from dvc.config import ConfigError
-from dvc.exceptions import DvcException
-from dvc.path_info import HTTPURLInfo, WebDAVURLInfo
-from dvc.progress import Tqdm
+from dvc.path_info import WebDAVURLInfo
 from dvc.scheme import Schemes
 
-from .base import BaseFileSystem
+from .fsspec_wrapper import FSSpecWrapper
 from .http import ask_password
 
 logger = logging.getLogger(__name__)
 
 
-class WebDAVConnectionError(DvcException):
-    def __init__(self, host):
-        super().__init__(f"Unable to connect to WebDAV {host}.")
-
-
-class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
-    # Use webdav scheme
+class WebDAVFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
     scheme = Schemes.WEBDAV
-
-    # URLInfo for Webdav ~ replaces webdav -> http
     PATH_CLS = WebDAVURLInfo
-
-    # Traversable as walk_files is implemented
     CAN_TRAVERSE = True
-
-    # Length of walk_files prefix
     TRAVERSE_PREFIX_LEN = 2
-
-    # Implementation based on webdav3.client
-    REQUIRES = {"webdavclient3": "webdav3.client"}
-
-    # Chunk size for buffered upload/download with progress bar
-    CHUNK_SIZE = 2 ** 16
-
+    REQUIRES = {"webdav4": "webdav4"}
     PARAM_CHECKSUM = "etag"
     DETAIL_FIELDS = frozenset(("etag", "size"))
 
-    # Constructor
-    def __init__(self, repo, config):
-        # Call BaseFileSystem constructor
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
-        # Get username from configuration
-        self.user = config.get("user", None)
+        cert_path = config.get("cert_path", None)
+        key_path = config.get("key_path", None)
+        self.prefix = config.get("prefix", "")
+        self.fs_args.update(
+            {
+                "base_url": config["url"],
+                "cert": cert_path if not key_path else (cert_path, key_path),
+                "verify": config.get("ssl_verify", True),
+                "timeout": config.get("timeout", 30),
+            }
+        )
 
-        # Get password from configuration (might be None ~ not set)
-        self.password = config.get("password", None)
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        path_info = WebDAVURLInfo(urlpath)
+        return {
+            "prefix": path_info.path.rstrip("/"),
+            "host": path_info.replace(path="").url,
+            "url": path_info.url.rstrip("/"),
+            "user": path_info.user,
+        }
 
-        # Whether to ask for password if it is not set
-        self.ask_password = config.get("ask_password", False)
+    def _prepare_credentials(self, **config):
+        user = config.get("user", None)
+        password = config.get("password", None)
 
-        # Use token for webdav auth
-        self.token = config.get("token", None)
+        headers = {}
+        token = config.get("token")
+        auth = None
+        if token:
+            headers.update({"Authorization": f"Bearer {token}"})
+        elif user:
+            if not password and config.get("ask_password"):
+                password = ask_password(config["host"], user)
+            auth = (user, password)
 
-        # Path to certificate
-        self.cert_path = config.get("cert_path", None)
+        return {"headers": headers, "auth": auth}
 
-        # Path to private key
-        self.key_path = config.get("key_path", None)
-
-        # Connection timeout
-        self.timeout = config.get("timeout", 30)
-
-        # Get URL from configuration
-        self.url = config.get("url", None)
-
-        # If URL in config parse path_info
-        if self.url:
-            self.path_info = self.PATH_CLS(self.url)
-
-            # If username not specified try to use from URL
-            if self.user is None and self.path_info.user is not None:
-                self.user = self.path_info.user
-
-            # Construct hostname from path_info by stripping path
-            http_info = HTTPURLInfo(self.path_info.url)
-            self.hostname = http_info.replace(path="").url
-
-    # Webdav client
     @wrap_prop(threading.Lock())
     @cached_property
-    def _client(self):
-        from webdav3.client import Client
+    def fs(self):
+        from webdav4.fsspec import WebdavFileSystem
 
-        # Set password or ask for it
-        if self.ask_password and self.password is None and self.token is None:
-            self.password = ask_password(self.hostname, self.user)
+        return WebdavFileSystem(**self.fs_args)
 
-        # Setup webdav client options dictionary
-        options = {
-            "webdav_hostname": self.hostname,
-            "webdav_login": self.user,
-            "webdav_password": self.password,
-            "webdav_token": self.token,
-            "webdav_cert_path": self.cert_path,
-            "webdav_key_path": self.key_path,
-            "webdav_timeout": self.timeout,
-            "webdav_chunk_size": self.CHUNK_SIZE,
-        }
+    def _upload_fobj(self, fobj, to_info, size: int = None):
+        rpath = self.translate_path_info(to_info)
+        self.makedirs(os.path.dirname(rpath))
+        # using upload_fileobj to directly upload fileobj rather than buffering
+        # and using overwrite=True to avoid check for an extra exists call,
+        # as caller should ensure that the file does not exist beforehand.
+        return self.fs.client.upload_fileobj(
+            fobj, rpath, overwrite=True, size=size
+        )
 
-        client = Client(options)
+    @lru_cache(512)
+    def translate_path_info(self, path):
+        if isinstance(path, self.PATH_CLS):
+            return path.path[len(self.prefix) :].lstrip("/")
+        return path
 
-        # Check whether client options are valid
-        if not client.valid():
-            raise ConfigError(
-                f"Configuration for WebDAV {self.hostname} is invalid."
-            )
+    _with_bucket = translate_path_info
 
-        # Check whether connection is valid (root should always exist)
-        if not client.check(self.path_info.path):
-            raise WebDAVConnectionError(self.hostname)
 
-        return client
-
-    def open(self, path_info, mode="r", encoding=None, **kwargs):
-        from webdav3.exceptions import RemoteResourceNotFound
-
-        assert mode in {"r", "rt", "rb"}
-
-        fobj = io.BytesIO()
-
-        try:
-            self._client.download_from(buff=fobj, remote_path=path_info.path)
-        except RemoteResourceNotFound as exc:
-            raise FileNotFoundError from exc
-
-        fobj.seek(0)
-
-        if "mode" == "rb":
-            return fobj
-
-        return io.TextIOWrapper(fobj, encoding=encoding)
-
-    # Checks whether file/directory exists at remote
-    def exists(self, path_info, use_dvcignore=True):
-        # Use webdav check to test for file existence
-        return self._client.check(path_info.path)
-
-    # Checks whether path points to directory
-    def isdir(self, path_info):
-        # Use webdav is_dir to test whether path points to a directory
-        return self._client.is_dir(path_info.path)
-
-    def walk_files(self, path_info, **kwargs):
-        if not self.exists(path_info):
-            return
-
-        for file in self.find(path_info):
-            yield path_info.replace(path=file)
-
-    def ls(self, path_info, detail=False):
-        for entry in self._client.list(path_info.path):
-            path = entry["path"]
-            if detail:
-                if entry["isdir"]:
-                    yield {"type": "directory", "name": path}
-                else:
-                    yield {
-                        "type": "file",
-                        "name": path,
-                        "size": entry["size"],
-                        "etag": entry["etag"],
-                    }
-            else:
-                yield path
-
-    def find(self, path_info, detail=False):
-        dirs = deque([path_info.path])
-
-        while dirs:
-            for entry in self._client.list(dirs.pop(), get_info=True):
-                path = entry["path"]
-                if entry["isdir"]:
-                    dirs.append(path)
-                    continue
-
-                if detail:
-                    yield {
-                        "type": "file",
-                        "name": path,
-                        "size": entry["size"],
-                        "etag": entry["etag"],
-                    }
-                else:
-                    yield path
-
-    # Removes file/directory
-    def remove(self, path_info):
-        # Use webdav client clean (DELETE) method to remove file/directory
-        self._client.clean(path_info.path)
-
-    # Creates directories
-    def makedirs(self, path_info):
-        # Terminate recursion
-        if path_info.path == self.path_info.path or self.exists(path_info):
-            return
-
-        # Recursively descent to root
-        self.makedirs(path_info.parent)
-
-        # Construct directory at current recursion depth
-        self._client.mkdir(path_info.path)
-
-    # Moves file/directory at remote
-    def move(self, from_info, to_info):
-        # Webdav client move
-        self._client.move(from_info.path, to_info.path)
-
-    def _upload_fobj(self, fobj, to_info):
-        # In contrast to other upload_fobj implementations, this one does not
-        # exactly do a chunked-upload but rather put everything in one request.
-        self.makedirs(to_info.parent)
-        self._client.upload_to(buff=fobj, remote_path=to_info.path)
-
-    # Downloads file from remote to file
-    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        # Progress from HTTPFileSystem
-        with open(to_file, "wb") as fd:
-            with Tqdm.wrapattr(
-                fd,
-                "write",
-                total=None if no_progress_bar else self.getsize(from_info),
-                leave=False,
-                desc=from_info.url if name is None else name,
-                disable=no_progress_bar,
-            ) as fd_wrapped:
-                # Download from WebDAV via buffer
-                self._client.download_from(
-                    buff=fd_wrapped, remote_path=from_info.path
-                )
-
-    # Uploads file to remote
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs,
-    ):
-        # First try to create parent directories
-        self.makedirs(to_info.parent)
-
-        file_size = os.path.getsize(from_file)
-        with open(from_file, "rb") as fd:
-            progress_context = (
-                nullcontext(fd)
-                if file_size == 0
-                else Tqdm.wrapattr(
-                    fd,
-                    "read",
-                    total=None if no_progress_bar else file_size,
-                    leave=False,
-                    desc=to_info.url if name is None else name,
-                    disable=no_progress_bar,
-                )
-            )
-            with progress_context as fd_wrapped:
-                self._client.upload_to(
-                    buff=fd_wrapped, remote_path=to_info.path
-                )
-
-    def info(self, path_info):
-        info = self._client.info(path_info.path)
-        return {
-            "size": int(info["size"]),
-            "etag": info["etag"],
-        }
+class WebDAVSFileSystem(WebDAVFileSystem):  # pylint:disable=abstract-method
+    scheme = Schemes.WEBDAVS

dvc/fs/webdavs.py

@@ -1,7 +0,0 @@
-from dvc.scheme import Schemes
-
-from .webdav import WebDAVFileSystem
-
-
-class WebDAVSFileSystem(WebDAVFileSystem):  # pylint:disable=abstract-method
-    scheme = Schemes.WEBDAVS

dvc/fs/webhdfs.py

@@ -4,7 +4,6 @@ import posixpath
 import shutil
 import threading
 from contextlib import contextmanager
-from urllib.parse import urlparse
 
 from funcy import cached_property, wrap_prop
 
@@ -30,33 +29,34 @@ def update_pbar(pbar, total):
     return update
 
 
-class WebHDFSFileSystem(BaseFileSystem):
+class WebHDFSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     scheme = Schemes.WEBHDFS
     PATH_CLS = CloudURLInfo
     REQUIRES = {"hdfs": "hdfs"}
     PARAM_CHECKSUM = "checksum"
     TRAVERSE_PREFIX_LEN = 2
 
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
+    def __init__(self, **config):
+        super().__init__(**config)
 
-        self.path_info = None
-        url = config.get("url")
-        if not url:
-            return
-
-        self.path_info = self.PATH_CLS(url)
-
-        parsed = urlparse(url)
-
-        self.host = parsed.hostname
-        self.user = parsed.username or config.get("user")
-        self.port = parsed.port
+        self.host = config["host"]
+        self.user = config.get("user")
+        self.port = config.get("port")
 
         self.hdfscli_config = config.get("hdfscli_config")
         self.token = config.get("webhdfs_token")
         self.alias = config.get("webhdfs_alias")
 
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        from fsspec.implementations.webhdfs import WebHDFS
+
+        return (
+            WebHDFS._get_kwargs_from_urls(  # pylint:disable=protected-access
+                urlpath
+            )
+        )
+
     @wrap_prop(threading.Lock())
     @cached_property
     def hdfs_client(self):
@@ -114,7 +114,7 @@ class WebHDFSFileSystem(BaseFileSystem):
 
         self.hdfs_client.delete(path_info.path)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         assert not isinstance(path_info, list)
         assert path_info.scheme == "webhdfs"
 
@@ -123,7 +123,7 @@ class WebHDFSFileSystem(BaseFileSystem):
 
     def info(self, path_info):
         st = self.hdfs_client.status(path_info.path)
-        return {"size": st["length"]}
+        return {"size": st["length"], "type": "file"}
 
     def checksum(self, path_info):
         return HashInfo(
@@ -141,7 +141,7 @@ class WebHDFSFileSystem(BaseFileSystem):
         self.hdfs_client.makedirs(to_info.parent.path)
         self.hdfs_client.rename(from_info.path, to_info.path)
 
-    def _upload_fobj(self, fobj, to_info):
+    def _upload_fobj(self, fobj, to_info, **kwargs):
         with self.hdfs_client.write(to_info.path) as fdest:
             shutil.copyfileobj(fobj, fdest)
 

dvc/hash_info.py

@@ -14,6 +14,7 @@ class HashInfo:
     value: Optional[str]
     size: Optional[int] = field(default=None, compare=False)
     nfiles: Optional[int] = field(default=None, compare=False)
+    obj_name: Optional[str] = field(default=None, compare=False)
 
     def __bool__(self):
         return bool(self.value)
@@ -21,6 +22,9 @@ class HashInfo:
     def __str__(self):
         return f"{self.name}: {self.value}"
 
+    def __hash__(self):
+        return hash((self.name, self.value))
+
     @classmethod
     def from_dict(cls, d):
         _d = d.copy() if d else {}

dvc/ignore.py

@@ -7,9 +7,12 @@ from itertools import groupby, takewhile
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
 
+from dvc.fs.base import BaseFileSystem
 from dvc.path_info import PathInfo
 from dvc.pathspec_math import PatternInfo, merge_patterns
+from dvc.scheme import Schemes
 from dvc.system import System
+from dvc.types import AnyPath, List, Optional
 from dvc.utils import relpath
 from dvc.utils.collections import PathStringTrie
 
@@ -49,16 +52,12 @@ class DvcIgnorePatterns(DvcIgnore):
         ]
 
     @classmethod
-    def from_files(cls, ignore_file_path, fs):
-        assert os.path.isabs(ignore_file_path)
-        dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-        ignore_file_rel_path = os.path.relpath(ignore_file_path, fs.fs_root)
-        with fs.open(ignore_file_path, encoding="utf-8") as fobj:
+    def from_file(cls, path, fs, name):
+        assert os.path.isabs(path)
+        dirname = os.path.normpath(os.path.dirname(path))
+        with fs.open(path, encoding="utf-8") as fobj:
             path_spec_lines = [
-                PatternInfo(
-                    line,
-                    "{}:{}:{}".format(ignore_file_rel_path, line_no + 1, line),
-                )
+                PatternInfo(line, f"{name}:{line_no + 1}:{line}")
                 for line_no, line in enumerate(
                     map(str.strip, fobj.readlines())
                 )
@@ -67,7 +66,7 @@ class DvcIgnorePatterns(DvcIgnore):
 
         return cls(path_spec_lines, dirname)
 
-    def __call__(self, root, dirs, files):
+    def __call__(self, root: List[str], dirs: List[str], files: List[str]):
         files = [f for f in files if not self.matches(root, f)]
         dirs = [d for d in dirs if not self.matches(root, d, True)]
 
@@ -159,26 +158,6 @@ def _no_match(path):
     return CheckIgnoreResult(path, False, ["::"])
 
 
-class DvcIgnoreFilterNoop:
-    def __init__(self, fs, root_dir):
-        pass
-
-    def __call__(self, root, dirs, files, **kwargs):
-        return dirs, files
-
-    def is_ignored_dir(self, _):
-        return False
-
-    def is_ignored_file(self, _):
-        return False
-
-    def check_ignore(self, path):
-        return _no_match(path)
-
-    def is_ignored(self, _):
-        return False
-
-
 class DvcIgnoreFilter:
     def __init__(self, fs, root_dir):
         from dvc.repo import Repo
@@ -187,33 +166,40 @@ class DvcIgnoreFilter:
             ".hg/",
             ".git/",
             ".git",
-            "{}/".format(Repo.DVC_DIR),
+            f"{Repo.DVC_DIR}/",
         ]
 
         self.fs = fs
         self.root_dir = root_dir
         self.ignores_trie_fs = PathStringTrie()
+        self._ignores_trie_subrepos = PathStringTrie()
         self.ignores_trie_fs[root_dir] = DvcIgnorePatterns(
             default_ignore_patterns, root_dir
         )
-        self._ignored_subrepos = PathStringTrie()
-        self._update(self.root_dir)
+        self._ignores_trie_subrepos[root_dir] = self.ignores_trie_fs[root_dir]
+        self._update(
+            self.root_dir,
+            self._ignores_trie_subrepos,
+            dnames=None,
+            ignore_subrepos=False,
+        )
+        self._update(
+            self.root_dir,
+            self.ignores_trie_fs,
+            dnames=None,
+            ignore_subrepos=True,
+        )
 
-    def _update(self, dirname):
-        self._update_sub_repo(dirname)
-
-        old_pattern = self.ignores_trie_fs.longest_prefix(dirname).value
+    def _update_trie(self, dirname: str, trie: PathStringTrie) -> None:
+        old_pattern = trie.longest_prefix(dirname).value
         matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
 
-        ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if not matches and self.fs.exists(
-            ignore_file_path, use_dvcignore=False
-        ):
-            new_pattern = DvcIgnorePatterns.from_files(
-                ignore_file_path, self.fs
-            )
+        path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
+        if not matches and self.fs.exists(path):
+            name = os.path.relpath(path, self.root_dir)
+            new_pattern = DvcIgnorePatterns.from_file(path, self.fs, name)
             if old_pattern:
-                self.ignores_trie_fs[dirname] = DvcIgnorePatterns(
+                trie[dirname] = DvcIgnorePatterns(
                     *merge_patterns(
                         old_pattern.pattern_list,
                         old_pattern.dirname,
@@ -222,11 +208,32 @@ class DvcIgnoreFilter:
                     )
                 )
             else:
-                self.ignores_trie_fs[dirname] = new_pattern
+                trie[dirname] = new_pattern
         elif old_pattern:
-            self.ignores_trie_fs[dirname] = old_pattern
+            trie[dirname] = old_pattern
 
-    def _update_sub_repo(self, path):
+    def _update(
+        self,
+        dirname: str,
+        ignore_trie: PathStringTrie,
+        dnames: Optional["List"],
+        ignore_subrepos: bool,
+    ) -> None:
+        self._update_trie(dirname, ignore_trie)
+
+        if ignore_subrepos:
+            if dnames is None:
+                try:
+                    _, dnames, _ = next(self.fs.walk(dirname))
+                except StopIteration:
+                    dnames = []
+
+            for dname in dnames:
+                self._update_sub_repo(
+                    os.path.join(dirname, dname), ignore_trie
+                )
+
+    def _update_sub_repo(self, path, ignore_trie: PathStringTrie):
         from dvc.repo import Repo
 
         if path == self.root_dir:
@@ -237,14 +244,11 @@ class DvcIgnoreFilter:
             return
 
         root, dname = os.path.split(path)
-        self._ignored_subrepos[root] = self._ignored_subrepos.get(
-            root, set()
-        ) | {dname}
         pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
         new_pattern = DvcIgnorePatterns([pattern_info], root)
-        old_pattern = self.ignores_trie_fs.longest_prefix(root).value
+        old_pattern = ignore_trie.longest_prefix(root).value
         if old_pattern:
-            self.ignores_trie_fs[root] = DvcIgnorePatterns(
+            ignore_trie[root] = DvcIgnorePatterns(
                 *merge_patterns(
                     old_pattern.pattern_list,
                     old_pattern.dirname,
@@ -253,25 +257,50 @@ class DvcIgnoreFilter:
                 )
             )
         else:
-            self.ignores_trie_fs[root] = new_pattern
+            ignore_trie[root] = new_pattern
 
     def __call__(self, root, dirs, files, ignore_subrepos=True):
-        for dname in dirs:
-            self._update_sub_repo(os.path.join(root, dname))
-
-        ignore_pattern = self._get_trie_pattern(root)
+        abs_root = os.path.abspath(root)
+        ignore_pattern = self._get_trie_pattern(
+            abs_root, dnames=dirs, ignore_subrepos=ignore_subrepos
+        )
         if ignore_pattern:
-            dirs, files = ignore_pattern(root, dirs, files)
-            if not ignore_subrepos:
-                dirs.extend(self._ignored_subrepos.get(root, []))
+            dirs, files = ignore_pattern(abs_root, dirs, files)
         return dirs, files
 
-    def _get_trie_pattern(self, dirname):
-        ignore_pattern = self.ignores_trie_fs.get(dirname)
+    def walk(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
+        ignore_subrepos = kwargs.pop("ignore_subrepos", True)
+        if fs.scheme == Schemes.LOCAL:
+            for root, dirs, files in fs.walk(path_info, **kwargs):
+                dirs[:], files[:] = self(
+                    root, dirs, files, ignore_subrepos=ignore_subrepos
+                )
+                yield root, dirs, files
+        else:
+            yield from fs.walk(path_info, **kwargs)
+
+    def walk_files(self, fs: BaseFileSystem, path_info: AnyPath, **kwargs):
+        if fs.scheme == Schemes.LOCAL:
+            for root, _, files in self.walk(fs, path_info, **kwargs):
+                for file in files:
+                    # NOTE: os.path.join is ~5.5 times slower
+                    yield PathInfo(f"{root}{os.sep}{file}")
+        else:
+            yield from fs.walk_files(path_info)
+
+    def _get_trie_pattern(
+        self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True
+    ) -> Optional["DvcIgnorePatterns"]:
+        if ignore_subrepos:
+            ignores_trie = self.ignores_trie_fs
+        else:
+            ignores_trie = self._ignores_trie_subrepos
+
+        ignore_pattern = ignores_trie.get(dirname)
         if ignore_pattern:
             return ignore_pattern
 
-        prefix = self.ignores_trie_fs.longest_prefix(dirname).key
+        prefix = ignores_trie.longest_prefix(dirname).key
         if not prefix:
             # outside of the repo
             return None
@@ -286,33 +315,31 @@ class DvcIgnoreFilter:
         dirs.append(dirname)
 
         for parent in dirs:
-            self._update(parent)
+            self._update(parent, ignores_trie, dnames, ignore_subrepos)
 
-        return self.ignores_trie_fs.get(dirname)
+        return ignores_trie.get(dirname)
 
-    def _is_ignored(self, path, is_dir=False):
+    def _is_ignored(
+        self, path: str, is_dir: bool = False, ignore_subrepos: bool = True
+    ):
         if self._outside_repo(path):
             return False
         dirname, basename = os.path.split(os.path.normpath(path))
-        ignore_pattern = self._get_trie_pattern(dirname)
+        ignore_pattern = self._get_trie_pattern(dirname, None, ignore_subrepos)
         if ignore_pattern:
             return ignore_pattern.matches(dirname, basename, is_dir)
         return False
 
-    def _is_subrepo(self, path):
-        dirname, basename = os.path.split(os.path.normpath(path))
-        return basename in self._ignored_subrepos.get(dirname, set())
-
-    def is_ignored_dir(self, path, ignore_subrepos=True):
+    def is_ignored_dir(self, path: str, ignore_subrepos: bool = True) -> bool:
+        "Only used in LocalFileSystem"
         path = os.path.abspath(path)
-        if not ignore_subrepos:
-            return not self._is_subrepo(path)
         if path == self.root_dir:
             return False
 
-        return self._is_ignored(path, True)
+        return self._is_ignored(path, True, ignore_subrepos=ignore_subrepos)
 
-    def is_ignored_file(self, path):
+    def is_ignored_file(self, path: str) -> bool:
+        "Only used in LocalFileSystem"
         path = os.path.abspath(path)
         return self._is_ignored(path, False)
 
@@ -339,21 +366,27 @@ class DvcIgnoreFilter:
             pattern = self._get_trie_pattern(dirname)
             if pattern:
                 matches = pattern.matches(
-                    dirname, basename, os.path.isdir(full_target), True,
+                    dirname, basename, os.path.isdir(full_target), True
                 )
 
                 if matches:
                     return CheckIgnoreResult(target, True, matches)
         return _no_match(target)
 
-    def is_ignored(self, path):
+    def is_ignored(
+        self, fs: BaseFileSystem, path: str, ignore_subrepos: bool = True
+    ) -> bool:
         # NOTE: can't use self.check_ignore(path).match for now, see
         # https://github.com/iterative/dvc/issues/4555
-        if os.path.isfile(path):
+        if fs.scheme != Schemes.LOCAL:
+            return False
+        if fs.isfile(path):
             return self.is_ignored_file(path)
-        if os.path.isdir(path):
-            return self.is_ignored_dir(path)
-        return self.is_ignored_file(path) or self.is_ignored_dir(path)
+        if fs.isdir(path):
+            return self.is_ignored_dir(path, ignore_subrepos)
+        return self.is_ignored_file(path) or self.is_ignored_dir(
+            path, ignore_subrepos
+        )
 
 
 def init(path):

dvc/info.py

@@ -20,6 +20,11 @@ if PKG is None:
 else:
     package = f"({PKG})"
 
+try:
+    import importlib.metadata as importlib_metadata
+except ImportError:  # < 3.8
+    import importlib_metadata  # type: ignore[no-redef]
+
 
 def get_dvc_info():
     info = [
@@ -27,38 +32,34 @@ def get_dvc_info():
         "---------------------------------",
         f"Platform: Python {platform.python_version()} on "
         f"{platform.platform()}",
-        f"Supports: {_get_supported_remotes()}",
+        f"Supports:{_get_supported_remotes()}",
     ]
 
     try:
-        repo = Repo()
+        with Repo() as repo:
+            # cache_dir might not exist yet (e.g. after `dvc init`), and we
+            # can't auto-create it, as it might cause issues if the user
+            # later decides to enable shared cache mode with
+            # `dvc config cache.shared group`.
+            if os.path.exists(repo.odb.local.cache_dir):
+                info.append(f"Cache types: {_get_linktype_support_info(repo)}")
+                fs_type = get_fs_type(repo.odb.local.cache_dir)
+                info.append(f"Cache directory: {fs_type}")
+            else:
+                info.append("Cache types: " + error_link("no-dvc-cache"))
 
-        # cache_dir might not exist yet (e.g. after `dvc init`), and we
-        # can't auto-create it, as it might cause issues if the user
-        # later decides to enable shared cache mode with
-        # `dvc config cache.shared group`.
-        if os.path.exists(repo.odb.local.cache_dir):
-            info.append(
-                "Cache types: {}".format(_get_linktype_support_info(repo))
-            )
-            fs_type = get_fs_type(repo.odb.local.cache_dir)
-            info.append(f"Cache directory: {fs_type}")
-        else:
-            info.append("Cache types: " + error_link("no-dvc-cache"))
-
-        info.append(f"Caches: {_get_caches(repo.odb)}")
-
-        info.append(f"Remotes: {_get_remotes(repo.config)}")
+            info.append(f"Caches: {_get_caches(repo.odb)}")
+            info.append(f"Remotes: {_get_remotes(repo.config)}")
 
+            root_directory = repo.root_dir
+            fs_root = get_fs_type(os.path.abspath(root_directory))
+            info.append(f"Workspace directory: {fs_root}")
+            info.append(f"Repo: {_get_dvc_repo_info(repo)}")
     except NotDvcRepoError:
         pass
     except SCMError:
         info.append("Repo: dvc, git (broken)")
-    else:
-        root_directory = repo.root_dir
-        fs_root = get_fs_type(os.path.abspath(root_directory))
-        info.append(f"Workspace directory: {fs_root}")
-        info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
+
     return "\n".join(info)
 
 
@@ -115,19 +116,23 @@ def _get_linktype_support_info(repo):
 
 
 def _get_supported_remotes():
-
     supported_remotes = []
     for scheme, fs_cls in FS_MAP.items():
         if not fs_cls.get_missing_deps():
-            supported_remotes.append(scheme)
+            dependencies = []
+            for requirement in fs_cls.REQUIRES:
+                dependencies.append(
+                    f"{requirement} = "
+                    f"{importlib_metadata.version(requirement)}"
+                )
 
-    if len(supported_remotes) == len(FS_MAP):
-        return "All remotes"
+            remote_info = scheme
+            if dependencies:
+                remote_info += " (" + ", ".join(dependencies) + ")"
+            supported_remotes.append(remote_info)
 
-    if len(supported_remotes) == 1:
-        return supported_remotes
-
-    return ", ".join(supported_remotes)
+    assert len(supported_remotes) >= 1
+    return "\n\t" + ",\n\t".join(supported_remotes)
 
 
 def get_fs_type(path):