dvc: bump to 2.0.8

This commit fixes 3 things:
1. The path printed on the terminal is properly is quoted.  This was
problematic when the output is piped to `xdg-open` if the path contained
spaces or quotes or special characters.

2. The path is resolved. This is not an issue at all, but makes the
output nicer.

3. The path with which the browser is opened with uses relative path.
This is done to solve issue in WSL, that the absolute path is not
accessible outside the WSL for host programs as the linux filesystem
is available on `/wsl$` path.

.codecov.yml

@@ -1 +1,10 @@
-
+comment: false
+coverage:
+  status:
+    project:
+      default:
+        target: 85
+        threshold: 10%
+    patch:
+      default:
+        threshold: 25%

.coveragerc

@@ -1,10 +1,14 @@
 [run]
+branch = True
+source = dvc
 omit =
     *__main__.py
 
 [report]
 exclude_lines =
     if __name__ == .__main__.:
+    if typing.TYPE_CHECKING:
+    if TYPE_CHECKING:
     pass
     raise NotImplementedError
     # pragma: no cover

.github/ISSUE_TEMPLATE/bug_report.md

@@ -3,16 +3,62 @@ name: "\U0001F41B Bug Report"
 about: Create a bug report to help us improve DVC
 ---
 
-## Bug Report
+# Bug Report
 
-### Please provide information about your setup
+<!--
+## Issue name
 
-**Output of `dvc version`:**
+Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug. 
+
+Example: `repro: doesn't detect input changes`
+-->
+
+## Description
+
+<!--
+A clear and concise description of what the bug is.
+-->
+
+### Reproduce
+
+<!--
+Step list of how to reproduce the bug
+-->
+
+<!--
+Example:
+
+1. dvc init
+2. Copy dataset.zip to the directory
+3. dvc add dataset.zip
+4. dvc run -d dataset.zip -o model ./train.sh
+5. modify dataset.zip
+6. dvc repro
+-->
+
+### Expected
+
+<!--
+A clear and concise description of what you expect to happen.
+-->
+
+### Environment information
+
+<!--
+This is required to ensure that we can reproduce the bug.
+-->
+
+**Output of `dvc doctor`:**
 
 ```console
-$ dvc version
+$ dvc doctor
 ```
 
 **Additional Information (if any):**
 
+<!--
+Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
+
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
+If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
+-->

.github/mergify.yml

@@ -0,0 +1,9 @@
+pull_request_rules:
+  - name: backport patches to 1.11
+    conditions:
+      - base=master
+      - label=backport-1.11
+    actions:
+      backport:
+        branches:
+          - "1.11"

.github/release-drafter.yml

@@ -1,5 +1,49 @@
 # Config for https://github.com/apps/release-drafter
-branches:
-  - master
+name-template: "$RESOLVED_VERSION 🦉 "
+tag-template: "$RESOLVED_VERSION"
+categories:
+  - title: "🚀 New Features and Enhancements "
+    labels:
+      - "feature"
+      - "enhancement"
+      - "ui"
+  - title: "💣 Breaking Changes "
+    labels:
+      - "breaking-change"
+  - title: "🏇 Optimizations "
+    labels:
+      - "optimize"
+      - "performance"
+  - title: "🐛 Bug Fixes "
+    labels:
+      - "bugfix"
+      - "bug"
+  - title: "🔨 Maintenance "
+    labels:
+      - "maintenance"
+      - "refactoring"
+      - "chore"
+      - "build"
+      - "ci"
+change-template: "- $TITLE (#$NUMBER) @$AUTHOR"
+version-resolver:
+  major:
+    labels:
+      - "major"
+  minor:
+    labels:
+      - "minor"
+  patch:
+    labels:
+      - "patch"
+  default: patch
+exclude-labels:
+  - "skip-changelog"
 template: |
+  > Refer to https://dvc.org/doc/install for installation instructions.
+
+  ## Changes
+
   $CHANGES
+
+  Thanks again to $CONTRIBUTORS for the contributions! 🎉

.github/workflows/check-patch.yaml

@@ -1,19 +0,0 @@
-name: Lint Python
-
-on: [push, pull_request]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.8
-    - name: Install requirements
-      run: bash ./scripts/ci/install.sh
-    - name: Check Patch
-      run: ./scripts/ci/check_patch.sh

.github/workflows/packages.yaml

@@ -0,0 +1,187 @@
+name: Build packages
+on:
+  push:
+    branches:
+    - master
+  release:
+    types: [released, prereleased]
+jobs:
+  deb:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build deb
+      run: python scripts/build.py deb
+    - name: Upload deb artifact
+      uses: actions/upload-artifact@v2
+      with:
+        path: scripts/fpm/dvc*.deb
+        if-no-files-found: error
+    - name: Publish deb
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_content_type: binary/octet-stream
+  rpm:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build rpm
+      run: python scripts/build.py rpm
+    - name: Upload rpm artifact
+      uses: actions/upload-artifact@v2
+      with:
+        path: scripts/fpm/dvc*.rpm
+        if-no-files-found: error
+    - name: Publish rpm
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  osxpkg:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build
+      run: python scripts/build.py osxpkg
+    - name: Upload osxpkg artifact
+      uses: actions/upload-artifact@v2
+      with:
+        path: scripts/fpm/dvc*.pkg
+        if-no-files-found: error
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
+        asset_name: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_content_type: binary/octet-stream
+  exe:
+    runs-on: windows-2019
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+    - name: Pull images
+      run: |
+        dvc pull
+    - name: Build
+      run: python scripts\build.py exe
+      shell: cmd
+    - name: Upload exe artifact
+      uses: actions/upload-artifact@v2
+      with:
+        path: scripts/innosetup/dvc*.exe
+        if-no-files-found: error
+    - name: Publish
+      if: github.event_name == 'release'
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      uses: actions/[email protected]
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
+        asset_name: dvc-${{ github.event.release.tag_name }}.exe
+        asset_content_type: binary/octet-stream
+  pip:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Install
+      run: pip install ".[all,tests]"
+    - name: Build packages
+      run: ./scripts/build_package.sh
+    - name: Publish packages to PyPI
+      if: github.event_name == 'release'
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_TOKEN }}
+  snap:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/[email protected]
+    - name: Build the package
+      uses: snapcore/action-build@v1
+      id: snapcraft
+#    - name: Test the package
+#      run: |
+#        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
+#        dvc version
+#        sudo snap remove dvc
+    - name: Publish to the edge channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'push'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: edge,v2/edge
+    - name: Publish to the stable channel
+      uses: snapcore/action-publish@v1
+      if: ${{ github.event_name == 'release' && github.event.release.released }}
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: stable,v2/stable

.github/workflows/tests.yaml

@@ -0,0 +1,96 @@
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+env:
+  DVC_TEST: "true"
+  HOMEBREW_NO_AUTO_UPDATE: 1
+  SHELL: /bin/bash
+  GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Install requirements
+      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+    - name: install temporary dependencies
+      run: |
+        pip install git+https://github.com/isidentical/gdrivefs@service-account
+        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
+    - name: Check README
+      run: python setup.py checkdocs
+    - uses: pre-commit/[email protected]
+  tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, windows-2019, macos-10.15]
+        pyv: ["3.6", "3.7", "3.8", "3.9"]
+# macos 11.0 workers take too long to start up
+#        include:
+#          - os: "macos-11.0"
+#            pyv: "3.9"
+        exclude:
+          - os: macos-10.15
+            pyv: "3.9"
+
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python
+      uses: actions/[email protected]
+      with:
+        python-version: ${{ matrix.pyv }}
+    - name: get pip cache dir
+      id: pip-cache-dir
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: set pip cache
+      id: pip-cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache-dir.outputs.dir }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: install
+      if: steps.cache.pip-cache-dir.cache-hit != 'true'
+      run: |
+        pip install wheel
+        pip install -e ".[all,tests]"
+    - name: install temporary dependencies
+      run: |
+        pip install git+https://github.com/isidentical/gdrivefs@service-account
+        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
+    - name: setup git
+      run: |
+        git config --global user.email "[email protected]"
+        git config --global user.name "DVC Tester"
+    - name: setup gs creds
+      shell: bash
+      if: env.GS_CREDS_JSON != ''
+      run: |
+        mkdir -p scripts/ci
+        echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
+    - name: run tests
+      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+    - name: upload coverage report
+      uses: codecov/[email protected]
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: false

.gitignore

@@ -1,12 +1,12 @@
 neatlynx.conf
 neatlynx/__pycache__
 .idea
-cache
+/cache
 *.pyc
 .env/
 .env2.7/
+.python-version
 
-cache
 .dvc.conf.lock
 .DS_Store
 build/

.pre-commit-config.yaml

@@ -4,15 +4,11 @@ repos:
         language_version: python3
     repo: https://github.com/ambv/black
     rev: 19.10b0
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.1.1
-    hooks:
-      - id: seed-isort-config
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 4.3.21
+    rev: 5.6.4
   - hooks:
       - id: flake8
         language_version: python3
@@ -22,7 +18,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.9
+    rev: 3.8.4
+  - hooks:
+      -  id: mypy
+         language_version: python3
+         files: ^dvc/
+    repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v0.790'
   - repo: local
     hooks:
     - id: pylint
@@ -37,7 +39,7 @@ repos:
         id: beautysh
         language_version: python3
     repo: https://github.com/lovesegfault/beautysh
-    rev: master
+    rev: 6.0.1
   - hooks:
       - args:
           - git-hook

.pylintrc

@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
-extension-pkg-whitelist=
+extension-pkg-whitelist=pygit2
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
@@ -15,10 +15,13 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook="import sys; \
-           from pathlib import Path; \
-           from pylint.config import find_pylintrc; \
-           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+init-hook="import sys;
+           from pathlib import Path;
+           from pylint.config import find_pylintrc;
+
+           tests_folder = Path(find_pylintrc()).parent / 'tests'
+           sys.path.append(str(tests_folder));
+           "
 
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
@@ -178,13 +181,17 @@ disable=format,
         # we use it to speedup/optimize
         import-outside-toplevel,
         fixme,
-
+        # we use cause of the exception for error message output
+        # so some of the times we don't use `raise from` to not show the cause
+        # of the error
+        raise-missing-from,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once). See also the "--disable" option for examples.
-enable=c-extension-no-member
+enable=c-extension-no-member,
+       no-else-return,
 
 
 [REPORTS]
@@ -265,7 +272,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
+generated-members=hdfs.*,pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -292,7 +299,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+ignored-modules=azure
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.

.restyled.yaml

@@ -1,20 +0,0 @@
----
-restylers:
-  - name: black
-    image: restyled/restyler-black:v19.10b0
-    command:
-    - black
-    arguments: []
-    include:
-    - "**/*.py"
-    interpreters:
-    - python
-  - name: isort
-    image: restyled/restyler-isort:v4.3.21
-    command:
-    - isort
-    arguments: []
-    include:
-    - "**/*.py"
-    interpreters:
-    - python

.travis.yml

@@ -1,162 +0,0 @@
-cache:
-  directories:
-    - $HOME/.cache/pip
-    - $HOME/.pyenv
-dist: xenial
-env:
-  global:
-    - HOMEBREW_NO_AUTO_UPDATE=1
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
-    - PIP_CACHE_DIR="$HOME/.cache/pip"  # unify pip cache location for all platforms
-stages:
-  - check
-  - test
-  - build
-# Travis doesn't support testing python on Mac yet, so we need
-# to workaround it by installing it directly with brew.
-#
-# Note that we specify language as ruby-2.4.3 to help us satisfy
-# fpm dependencies, as travis runs el capitan with an old ruby
-# version, that causes fpm to crash.
-#
-# Each job will use the previous job's `stage` tag if not specified.
-jobs:
-  include:
-  # patch formatting checks
-  - name: "Check patch formatting"
-    stage: check
-    os: linux
-    language: python
-    python: 3.7
-    before_install: bash ./scripts/ci/install.sh
-    install:
-    script: ./scripts/ci/check_patch.sh
-  # test jobs
-  - name: "3.7 on Windows"
-    stage: test
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "3.8 on Windows"
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.8.0
-      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
-  - name: "3.6 on Linux"
-    language: python
-    python: 3.6
-  - name: "3.7 on Linux"
-    language: python
-    python: 3.7
-  - name: "3.8 on Linux"
-    language: python
-    python: 3.8
-  - name: "3.7 on Mac"
-    os: osx
-    osx_image: xcode11.3
-  # build jobs
-  - name: "Mac pkg"
-    stage: build
-    os: osx
-    osx_image: xcode9.4
-    language: ruby
-    rvm: 2.4.3
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "Windows pkg"
-    os: windows
-    language: shell
-    install:
-    script: ./scripts/build_windows.cmd
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "Linux pkgs"
-    language: python
-    python: 3.7
-    before_install:
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "PyPI pkgs"
-    language: python
-    python: 3.8
-    script: ./scripts/build_package.sh
-  - name: Snapcraft snap
-    language: python
-    python: 3.8
-    addons:
-     snaps:
-     - name: snapcraft
-       channel: stable
-       confinement: classic
-     - name: review-tools
-       channel: stable
-     - name: lxd
-       channel: stable
-    env:
-    - SNAPCRAFT_IMAGE_INFO: |
-        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
-    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
-    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
-    install:
-    - sudo usermod --append --groups lxd $USER
-    - sudo /snap/bin/lxd.migrate -yes
-    - sudo /snap/bin/lxd waitready
-    - sudo /snap/bin/lxd init --auto
-    script:
-    - ./scripts/build_snap.sh
-    after_failure:
-    - sudo journalctl -u snapd
-
-before_install:
-  - bash ./scripts/ci/before_install.sh
-  - source env.sh
-  - bash ./scripts/ci/check_python.sh
-install:
-  - bash ./scripts/ci/install.sh
-script:
-  - bash ./scripts/ci/script.sh
-deploy:
-  - provider: releases
-    api_key:
-      secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
-    file_glob: true
-    file:
-      - dvc*.rpm
-      - dvc*.deb
-      - dvc*.pkg
-      - dvc*.exe
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: pypi
-    user:
-      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-    password:
-      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-    skip_existing: true
-    file_glob: true
-    file:
-      - dist/dvc*.whl
-      - dist/dvc*.tar.gz
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: script
-    skip_cleanup: true
-    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
-    on:
-      all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
-      repo: iterative/dvc
-      stage: build

README.rst

@@ -201,38 +201,6 @@ Contributing
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
-   :alt: 0
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
-   :alt: 1
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
-   :alt: 2
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
-   :alt: 3
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
-   :alt: 4
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
-   :alt: 5
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
-   :alt: 6
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
-   :alt: 7
-
 Mailing List
 ============
 
@@ -254,6 +222,9 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
+Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA.
+
+
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png
    :target: https://dvc.org
    :alt: DVC logo
@@ -262,9 +233,9 @@ Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
    :target: https://travis-ci.com/iterative/dvc/branches
    :alt: Release
 
-.. |CI| image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev&logo=travis
-   :target: https://travis-ci.com/iterative/dvc/builds
-   :alt: Travis dev branch
+.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
+   :target: https://github.com/iterative/dvc/actions
+   :alt: GHA Tests
 
 .. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
    :target: https://codeclimate.com/github/iterative/dvc

dvc/_debug.py

@@ -0,0 +1,91 @@
+from contextlib import ExitStack, contextmanager
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from argparse import Namespace
+
+
+@contextmanager
+def instrument(html_output=False):
+    """Run a statistical profiler"""
+    try:
+        from pyinstrument import Profiler  # pylint: disable=import-error
+    except ImportError:
+        print("Failed to run profiler, pyinstrument is not installed")
+        yield
+        return
+
+    profiler = Profiler()
+
+    profiler.start()
+    yield
+    profiler.stop()
+
+    if html_output:
+        profiler.open_in_browser()
+        return
+    print(profiler.output_text(unicode=True, color=True))
+
+
+@contextmanager
+def profile(dump_path: str = None):
+    """Run a cprofile"""
+    import cProfile
+
+    prof = cProfile.Profile()
+    prof.enable()
+
+    yield
+
+    prof.disable()
+    if not dump_path:
+        prof.print_stats(sort="cumtime")
+        return
+    prof.dump_stats(dump_path)
+
+
+@contextmanager
+def debug():
+    try:
+        yield
+    except Exception:  # pylint: disable=broad-except
+        try:
+            import ipdb as pdb  # noqa: T100, pylint: disable=import-error
+        except ImportError:
+            import pdb  # noqa: T100
+        pdb.post_mortem()
+
+        raise  # prevent from jumping ahead
+
+
+@contextmanager
+def debugtools(args: "Namespace" = None, **kwargs):
+    kw = vars(args) if args else {}
+    kw.update(kwargs)
+
+    with ExitStack() as stack:
+        if kw.get("pdb"):
+            stack.enter_context(debug())
+        if kw.get("cprofile") or kw.get("cprofile_dump"):
+            stack.enter_context(profile(kw.get("cprofile_dump")))
+        if kw.get("instrument") or kw.get("instrument_open"):
+            stack.enter_context(instrument(kw.get("instrument_open", False)))
+        yield
+
+
+def add_debugging_flags(parser):
+    from argparse import SUPPRESS
+
+    parser.add_argument(
+        "--cprofile", action="store_true", default=False, help=SUPPRESS,
+    )
+    parser.add_argument("--cprofile-dump", help=SUPPRESS)
+    parser.add_argument(
+        "--pdb", action="store_true", default=False, help=SUPPRESS,
+    )
+    parser.add_argument(
+        "--instrument", action="store_true", default=False, help=SUPPRESS,
+    )
+    parser.add_argument(
+        "--instrument-open", action="store_true", default=False, help=SUPPRESS,
+    )

dvc/analytics.py

@@ -1,24 +1,6 @@
 import json
 import logging
 import os
-import platform
-import sys
-import tempfile
-import uuid
-
-import distro
-import requests
-
-from dvc import __version__
-from dvc.config import Config, to_bool
-from dvc.daemon import daemon
-from dvc.exceptions import NotDvcRepoError
-from dvc.lock import Lock, LockError
-from dvc.repo import Repo
-from dvc.scm import SCM, NoSCM
-from dvc.scm.base import SCMError
-from dvc.utils import env2bool, is_binary
-from dvc.utils.fs import makedirs
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +17,10 @@ def collect_and_send_report(args=None, return_code=None):
     report as a JSON, where the _collector_ generates it and the _sender_
     removes it after sending it.
     """
+    import tempfile
+
+    from dvc.daemon import daemon
+
     report = {}
 
     # Include command execution information on the report only when available.
@@ -50,6 +36,9 @@ def collect_and_send_report(args=None, return_code=None):
 
 
 def is_enabled():
+    from dvc.config import Config, to_bool
+    from dvc.utils import env2bool
+
     if env2bool("DVC_TEST"):
         return False
 
@@ -69,6 +58,8 @@ def send(path):
     `collect_and_send_report`. Sending happens on another process,
     thus, the need of removing such file afterwards.
     """
+    import requests
+
     url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
@@ -86,6 +77,11 @@ def send(path):
 
 
 def _scm_in_use():
+    from dvc.exceptions import NotDvcRepoError
+    from dvc.repo import Repo
+    from dvc.scm import SCM, NoSCM
+    from dvc.scm.base import SCMError
+
     try:
         scm = SCM(root_dir=Repo.find_root())
         return type(scm).__name__
@@ -99,6 +95,9 @@ def _runtime_info():
     """
     Gather information from the environment where DVC runs to fill a report.
     """
+    from dvc import __version__
+    from dvc.utils import is_binary
+
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
@@ -109,6 +108,11 @@ def _runtime_info():
 
 
 def _system_info():
+    import platform
+    import sys
+
+    import distro
+
     system = platform.system()
 
     if system == "Windows":
@@ -147,6 +151,12 @@ def _find_or_create_user_id():
 
     IDs are generated randomly with UUID.
     """
+    import uuid
+
+    from dvc.config import Config
+    from dvc.lock import Lock, LockError
+    from dvc.utils.fs import makedirs
+
     config_dir = Config.get_dir("global")
     fname = os.path.join(config_dir, "user_id")
     lockfile = os.path.join(config_dir, "user_id.lock")

dvc/api.py

@@ -1,36 +1,35 @@
 import os
 from contextlib import _GeneratorContextManager as GCM
-from contextlib import contextmanager
 
-from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.external_repo import external_repo
+from funcy import reraise
+
+from dvc.exceptions import OutputNotFoundError, PathMissingError
+from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
 
-class UrlNotDvcRepoError(DvcException):
-    """Thrown if the given URL is not a DVC repository."""
-
-    def __init__(self, url):
-        super().__init__(f"'{url}' is not a DVC repository.")
-
-
 def get_url(path, repo=None, rev=None, remote=None):
     """
     Returns the URL to the storage location of a data file or directory tracked
     in a DVC repo. For Git repos, HEAD is used unless a rev argument is
     supplied. The default remote is tried unless a remote argument is supplied.
 
-    Raises UrlNotDvcRepoError if repo is not a DVC project.
+    Raises OutputNotFoundError if the file is not tracked by DVC.
 
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
     """
-    with _make_repo(repo, rev=rev) as _repo:
-        if not isinstance(_repo, Repo):
-            raise UrlNotDvcRepoError(_repo.url)  # pylint: disable=no-member
-        out = _repo.find_out_by_relpath(path)
-        remote_obj = _repo.cloud.get_remote(remote)
-        return str(remote_obj.tree.hash_to_path_info(out.checksum))
+    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
+        path_info = PathInfo(_repo.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, repo)):
+            metadata = _repo.repo_fs.metadata(path_info)
+
+        if not metadata.is_dvc:
+            raise OutputNotFoundError(path, repo)
+
+        cloud = metadata.repo.cloud
+        md5 = metadata.repo.dvcfs.info(path_info)["md5"]
+        return cloud.get_url_for(remote, checksum=md5)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin
@@ -73,7 +72,7 @@ class _OpenContextManager(GCM):
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    with _make_repo(repo, rev=rev) as _repo:
+    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
         with _repo.open_by_relpath(
             path, remote=remote, mode=mode, encoding=encoding
         ) as fd:
@@ -92,14 +91,33 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
         return fd.read()
 
 
-@contextmanager
-def _make_repo(repo_url=None, rev=None):
-    repo_url = repo_url or os.getcwd()
-    if rev is None and os.path.exists(repo_url):
-        try:
-            yield Repo(repo_url)
-            return
-        except NotDvcRepoError:
-            pass  # fallthrough to external_repo
-    with external_repo(url=repo_url, rev=rev) as repo:
-        yield repo
+def make_checkpoint():
+    """
+    Signal DVC to create a checkpoint experiment.
+
+    If the current process is being run from DVC, this function will block
+    until DVC has finished creating the checkpoint. Otherwise, this function
+    will return immediately.
+    """
+    import builtins
+    from time import sleep
+
+    from dvc.env import DVC_CHECKPOINT, DVC_ROOT
+    from dvc.stage.monitor import CheckpointTask
+
+    if os.getenv(DVC_CHECKPOINT) is None:
+        return
+
+    root_dir = os.getenv(DVC_ROOT, Repo.find_root())
+    signal_file = os.path.join(
+        root_dir, Repo.DVC_DIR, "tmp", CheckpointTask.SIGNAL_FILE
+    )
+
+    with builtins.open(signal_file, "w") as fobj:
+        # NOTE: force flushing/writing empty file to disk, otherwise when
+        # run in certain contexts (pytest) file may not actually be written
+        fobj.write("")
+        fobj.flush()
+        os.fsync(fobj.fileno())
+    while os.path.exists(signal_file):
+        sleep(0.1)

dvc/cache/base.py

@@ -1,554 +0,0 @@
-import json
-import logging
-from copy import copy
-
-from shortuuid import uuid
-
-import dvc.prompt as prompt
-from dvc.exceptions import CheckoutError, ConfirmRemoveError, DvcException
-from dvc.path_info import WindowsPathInfo
-from dvc.progress import Tqdm
-from dvc.remote.slow_link_detection import slow_link_guard
-
-from ..tree.base import RemoteActionNotImplemented
-
-logger = logging.getLogger(__name__)
-
-STATUS_OK = 1
-STATUS_MISSING = 2
-STATUS_NEW = 3
-STATUS_DELETED = 4
-
-STATUS_MAP = {
-    # (local_exists, remote_exists)
-    (True, True): STATUS_OK,
-    (False, False): STATUS_MISSING,
-    (True, False): STATUS_NEW,
-    (False, True): STATUS_DELETED,
-}
-
-
-class DirCacheError(DvcException):
-    def __init__(self, hash_):
-        super().__init__(
-            f"Failed to load dir cache for hash value: '{hash_}'."
-        )
-
-
-class CloudCache:
-    """Cloud cache class."""
-
-    DEFAULT_CACHE_TYPES = ["copy"]
-    CACHE_MODE = None
-
-    def __init__(self, tree):
-        self.tree = tree
-        self.repo = tree.repo
-
-        self.cache_types = tree.config.get("type") or copy(
-            self.DEFAULT_CACHE_TYPES
-        )
-        self.cache_type_confirmed = False
-        self._dir_info = {}
-
-    def get_dir_cache(self, hash_):
-        assert hash_
-
-        dir_info = self._dir_info.get(hash_)
-        if dir_info:
-            return dir_info
-
-        try:
-            dir_info = self.load_dir_cache(hash_)
-        except DirCacheError:
-            dir_info = []
-
-        self._dir_info[hash_] = dir_info
-        return dir_info
-
-    def load_dir_cache(self, hash_):
-        path_info = self.tree.hash_to_path_info(hash_)
-
-        try:
-            with self.tree.open(path_info, "r") as fobj:
-                d = json.load(fobj)
-        except (ValueError, FileNotFoundError) as exc:
-            raise DirCacheError(hash_) from exc
-
-        if not isinstance(d, list):
-            logger.error(
-                "dir cache file format error '%s' [skipping the file]",
-                path_info,
-            )
-            return []
-
-        if self.tree.PATH_CLS == WindowsPathInfo:
-            # only need to convert it for Windows
-            for info in d:
-                # NOTE: here is a BUG, see comment to .as_posix() below
-                relpath = info[self.tree.PARAM_RELPATH]
-                info[self.tree.PARAM_RELPATH] = relpath.replace(
-                    "/", self.tree.PATH_CLS.sep
-                )
-
-        return d
-
-    def changed(self, path_info, hash_info):
-        """Checks if data has changed.
-
-        A file is considered changed if:
-            - It doesn't exist on the working directory (was unlinked)
-            - Hash value is not computed (saving a new file)
-            - The hash value stored is different from the given one
-            - There's no file in the cache
-
-        Args:
-            path_info: dict with path information.
-            hash: expected hash value for this data.
-
-        Returns:
-            bool: True if data has changed, False otherwise.
-        """
-
-        logger.debug(
-            "checking if '%s'('%s') has changed.", path_info, hash_info
-        )
-
-        if not self.tree.exists(path_info):
-            logger.debug("'%s' doesn't exist.", path_info)
-            return True
-
-        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
-        if hash_ is None:
-            logger.debug("hash value for '%s' is missing.", path_info)
-            return True
-
-        if self.changed_cache(hash_):
-            logger.debug("cache for '%s'('%s') has changed.", path_info, hash_)
-            return True
-
-        actual = self.tree.get_hash(path_info)
-        if hash_ != actual:
-            logger.debug(
-                "hash value '%s' for '%s' has changed (actual '%s').",
-                hash_,
-                actual,
-                path_info,
-            )
-            return True
-
-        logger.debug("'%s' hasn't changed.", path_info)
-        return False
-
-    def link(self, from_info, to_info):
-        self._link(from_info, to_info, self.cache_types)
-
-    def _link(self, from_info, to_info, link_types):
-        assert self.tree.isfile(from_info)
-
-        self.tree.makedirs(to_info.parent)
-
-        self._try_links(from_info, to_info, link_types)
-
-    def _verify_link(self, path_info, link_type):
-        if self.cache_type_confirmed:
-            return
-
-        is_link = getattr(self.tree, f"is_{link_type}", None)
-        if is_link and not is_link(path_info):
-            self.tree.remove(path_info)
-            raise DvcException(f"failed to verify {link_type}")
-
-        self.cache_type_confirmed = True
-
-    @slow_link_guard
-    def _try_links(self, from_info, to_info, link_types):
-        while link_types:
-            link_method = getattr(self.tree, link_types[0])
-            try:
-                self._do_link(from_info, to_info, link_method)
-                self._verify_link(to_info, link_types[0])
-                return
-
-            except DvcException as exc:
-                logger.debug(
-                    "Cache type '%s' is not supported: %s", link_types[0], exc
-                )
-                del link_types[0]
-
-        raise DvcException("no possible cache types left to try out.")
-
-    def _do_link(self, from_info, to_info, link_method):
-        if self.tree.exists(to_info):
-            raise DvcException(f"Link '{to_info}' already exists!")
-
-        link_method(from_info, to_info)
-
-        logger.debug(
-            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
-        )
-
-    def _save_file(self, path_info, tree, hash_, save_link=True, **kwargs):
-        assert hash_
-
-        cache_info = self.tree.hash_to_path_info(hash_)
-        if tree == self.tree:
-            if self.changed_cache(hash_):
-                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
-                self.link(cache_info, path_info)
-            elif self.tree.iscopy(path_info) and self._cache_is_copy(
-                path_info
-            ):
-                # Default relink procedure involves unneeded copy
-                self.tree.unprotect(path_info)
-            else:
-                self.tree.remove(path_info)
-                self.link(cache_info, path_info)
-
-            if save_link:
-                self.tree.state.save_link(path_info)
-            # we need to update path and cache, since in case of reflink,
-            # or copy cache type moving original file results in updates on
-            # next executed command, which causes md5 recalculation
-            self.tree.state.save(path_info, hash_)
-        else:
-            if self.changed_cache(hash_):
-                with tree.open(path_info, mode="rb") as fobj:
-                    # if tree has fetch enabled, DVC out will be fetched on
-                    # open and we do not need to read/copy any data
-                    if not (
-                        tree.isdvc(path_info, strict=False) and tree.fetch
-                    ):
-                        self.tree.copy_fobj(fobj, cache_info)
-                callback = kwargs.get("download_callback")
-                if callback:
-                    callback(1)
-
-        self.tree.state.save(cache_info, hash_)
-        return {self.tree.PARAM_CHECKSUM: hash_}
-
-    def _cache_is_copy(self, path_info):
-        """Checks whether cache uses copies."""
-        if self.cache_type_confirmed:
-            return self.cache_types[0] == "copy"
-
-        if set(self.cache_types) <= {"copy"}:
-            return True
-
-        workspace_file = path_info.with_name("." + uuid())
-        test_cache_file = self.tree.path_info / ".cache_type_test_file"
-        if not self.tree.exists(test_cache_file):
-            with self.tree.open(test_cache_file, "wb") as fobj:
-                fobj.write(bytes(1))
-        try:
-            self.link(test_cache_file, workspace_file)
-        finally:
-            self.tree.remove(workspace_file)
-            self.tree.remove(test_cache_file)
-
-        self.cache_type_confirmed = True
-        return self.cache_types[0] == "copy"
-
-    def _save_dir(self, path_info, tree, hash_, save_link=True, **kwargs):
-        dir_info = self.get_dir_cache(hash_)
-        for entry in Tqdm(
-            dir_info, desc="Saving " + path_info.name, unit="file"
-        ):
-            entry_info = path_info / entry[self.tree.PARAM_RELPATH]
-            entry_hash = entry[self.tree.PARAM_CHECKSUM]
-            self._save_file(
-                entry_info, tree, entry_hash, save_link=False, **kwargs
-            )
-
-        if save_link:
-            self.tree.state.save_link(path_info)
-        if self.tree.exists(path_info):
-            self.tree.state.save(path_info, hash_)
-
-        cache_info = self.tree.hash_to_path_info(hash_)
-        self.tree.state.save(cache_info, hash_)
-        return {self.tree.PARAM_CHECKSUM: hash_}
-
-    def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
-        if path_info.scheme != self.tree.scheme:
-            raise RemoteActionNotImplemented(
-                f"save {path_info.scheme} -> {self.tree.scheme}",
-                self.tree.scheme,
-            )
-
-        if not hash_info:
-            hash_info = self.tree.save_info(path_info, tree=tree, **kwargs)
-        hash_ = hash_info[self.tree.PARAM_CHECKSUM]
-        return self._save(path_info, tree, hash_, save_link, **kwargs)
-
-    def _save(self, path_info, tree, hash_, save_link=True, **kwargs):
-        to_info = self.tree.hash_to_path_info(hash_)
-        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
-
-        if tree.isdir(path_info):
-            return self._save_dir(path_info, tree, hash_, save_link, **kwargs)
-        return self._save_file(path_info, tree, hash_, save_link, **kwargs)
-
-    # Override to return path as a string instead of PathInfo for clouds
-    # which support string paths (see local)
-    def hash_to_path(self, hash_):
-        return self.tree.hash_to_path_info(hash_)
-
-    def changed_cache_file(self, hash_):
-        """Compare the given hash with the (corresponding) actual one.
-
-        - Use `State` as a cache for computed hashes
-            + The entries are invalidated by taking into account the following:
-                * mtime
-                * inode
-                * size
-                * hash
-
-        - Remove the file from cache if it doesn't match the actual hash
-        """
-        # Prefer string path over PathInfo when possible due to performance
-        cache_info = self.hash_to_path(hash_)
-        if self.tree.is_protected(cache_info):
-            logger.debug(
-                "Assuming '%s' is unchanged since it is read-only", cache_info
-            )
-            return False
-
-        actual = self.tree.get_hash(cache_info)
-
-        logger.debug(
-            "cache '%s' expected '%s' actual '%s'", cache_info, hash_, actual,
-        )
-
-        if not hash_ or not actual:
-            return True
-
-        if actual.split(".")[0] == hash_.split(".")[0]:
-            # making cache file read-only so we don't need to check it
-            # next time
-            self.tree.protect(cache_info)
-            return False
-
-        if self.tree.exists(cache_info):
-            logger.warning("corrupted cache file '%s'.", cache_info)
-            self.tree.remove(cache_info)
-
-        return True
-
-    def _changed_dir_cache(self, hash_, path_info=None, filter_info=None):
-        if self.changed_cache_file(hash_):
-            return True
-
-        for entry in self.get_dir_cache(hash_):
-            entry_hash = entry[self.tree.PARAM_CHECKSUM]
-
-            if path_info and filter_info:
-                entry_info = path_info / entry[self.tree.PARAM_RELPATH]
-                if not entry_info.isin_or_eq(filter_info):
-                    continue
-
-            if self.changed_cache_file(entry_hash):
-                return True
-
-        return False
-
-    def changed_cache(self, hash_, path_info=None, filter_info=None):
-        if self.tree.is_dir_hash(hash_):
-            return self._changed_dir_cache(
-                hash_, path_info=path_info, filter_info=filter_info
-            )
-        return self.changed_cache_file(hash_)
-
-    def already_cached(self, path_info):
-        current = self.tree.get_hash(path_info)
-
-        if not current:
-            return False
-
-        return not self.changed_cache(current)
-
-    def safe_remove(self, path_info, force=False):
-        if not self.tree.exists(path_info):
-            return
-
-        if not force and not self.already_cached(path_info):
-            msg = (
-                "file '{}' is going to be removed."
-                " Are you sure you want to proceed?".format(str(path_info))
-            )
-
-            if not prompt.confirm(msg):
-                raise ConfirmRemoveError(str(path_info))
-
-        self.tree.remove(path_info)
-
-    def _checkout_file(
-        self, path_info, hash_, force, progress_callback=None, relink=False
-    ):
-        """The file is changed we need to checkout a new copy"""
-        added, modified = True, False
-        cache_info = self.tree.hash_to_path_info(hash_)
-        if self.tree.exists(path_info):
-            logger.debug("data '%s' will be replaced.", path_info)
-            self.safe_remove(path_info, force=force)
-            added, modified = False, True
-
-        self.link(cache_info, path_info)
-        self.tree.state.save_link(path_info)
-        self.tree.state.save(path_info, hash_)
-        if progress_callback:
-            progress_callback(str(path_info))
-
-        return added, modified and not relink
-
-    def _checkout_dir(
-        self,
-        path_info,
-        hash_,
-        force,
-        progress_callback=None,
-        relink=False,
-        filter_info=None,
-    ):
-        added, modified = False, False
-        # Create dir separately so that dir is created
-        # even if there are no files in it
-        if not self.tree.exists(path_info):
-            added = True
-            self.tree.makedirs(path_info)
-
-        dir_info = self.get_dir_cache(hash_)
-
-        logger.debug("Linking directory '%s'.", path_info)
-
-        for entry in dir_info:
-            relative_path = entry[self.tree.PARAM_RELPATH]
-            entry_hash = entry[self.tree.PARAM_CHECKSUM]
-            entry_cache_info = self.tree.hash_to_path_info(entry_hash)
-            entry_info = path_info / relative_path
-
-            if filter_info and not entry_info.isin_or_eq(filter_info):
-                continue
-
-            entry_hash_info = {self.tree.PARAM_CHECKSUM: entry_hash}
-            if relink or self.changed(entry_info, entry_hash_info):
-                modified = True
-                self.safe_remove(entry_info, force=force)
-                self.link(entry_cache_info, entry_info)
-                self.tree.state.save(entry_info, entry_hash)
-            if progress_callback:
-                progress_callback(str(entry_info))
-
-        modified = (
-            self._remove_redundant_files(path_info, dir_info, force)
-            or modified
-        )
-
-        self.tree.state.save_link(path_info)
-        self.tree.state.save(path_info, hash_)
-
-        # relink is not modified, assume it as nochange
-        return added, not added and modified and not relink
-
-    def _remove_redundant_files(self, path_info, dir_info, force):
-        existing_files = set(self.tree.walk_files(path_info))
-
-        needed_files = {
-            path_info / entry[self.tree.PARAM_RELPATH] for entry in dir_info
-        }
-        redundant_files = existing_files - needed_files
-        for path in redundant_files:
-            self.safe_remove(path, force)
-
-        return bool(redundant_files)
-
-    def checkout(
-        self,
-        path_info,
-        hash_info,
-        force=False,
-        progress_callback=None,
-        relink=False,
-        filter_info=None,
-    ):
-        if path_info.scheme not in ["local", self.tree.scheme]:
-            raise NotImplementedError
-
-        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
-        failed = None
-        skip = False
-        if not hash_:
-            logger.warning(
-                "No file hash info found for '%s'. " "It won't be created.",
-                path_info,
-            )
-            self.safe_remove(path_info, force=force)
-            failed = path_info
-
-        elif not relink and not self.changed(path_info, hash_info):
-            logger.debug("Data '%s' didn't change.", path_info)
-            skip = True
-
-        elif self.changed_cache(
-            hash_, path_info=path_info, filter_info=filter_info
-        ):
-            logger.warning(
-                "Cache '%s' not found. File '%s' won't be created.",
-                hash_,
-                path_info,
-            )
-            self.safe_remove(path_info, force=force)
-            failed = path_info
-
-        if failed or skip:
-            if progress_callback:
-                progress_callback(
-                    str(path_info),
-                    self.get_files_number(
-                        self.tree.path_info, hash_, filter_info
-                    ),
-                )
-            if failed:
-                raise CheckoutError([failed])
-            return
-
-        logger.debug("Checking out '%s' with cache '%s'.", path_info, hash_)
-
-        return self._checkout(
-            path_info, hash_, force, progress_callback, relink, filter_info,
-        )
-
-    def _checkout(
-        self,
-        path_info,
-        hash_,
-        force=False,
-        progress_callback=None,
-        relink=False,
-        filter_info=None,
-    ):
-        if not self.tree.is_dir_hash(hash_):
-            return self._checkout_file(
-                path_info, hash_, force, progress_callback, relink
-            )
-
-        return self._checkout_dir(
-            path_info, hash_, force, progress_callback, relink, filter_info
-        )
-
-    def get_files_number(self, path_info, hash_, filter_info):
-        from funcy.py3 import ilen
-
-        if not hash_:
-            return 0
-
-        if not self.tree.is_dir_hash(hash_):
-            return 1
-
-        if not filter_info:
-            return len(self.get_dir_cache(hash_))
-
-        return ilen(
-            filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
-            for entry in self.get_dir_cache(hash_)
-        )

dvc/cache/local.py

@@ -1,431 +0,0 @@
-import errno
-import logging
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial, wraps
-
-from funcy import cached_property, concat
-
-from dvc.exceptions import DownloadError, UploadError
-from dvc.path_info import PathInfo
-from dvc.progress import Tqdm
-
-from ..remote.base import index_locked
-from ..tree.local import LocalTree
-from .base import (
-    STATUS_DELETED,
-    STATUS_MAP,
-    STATUS_MISSING,
-    STATUS_NEW,
-    CloudCache,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def _log_exceptions(func, operation):
-    @wraps(func)
-    def wrapper(from_info, to_info, *args, **kwargs):
-        try:
-            func(from_info, to_info, *args, **kwargs)
-            return 0
-        except Exception as exc:  # pylint: disable=broad-except
-            # NOTE: this means we ran out of file descriptors and there is no
-            # reason to try to proceed, as we will hit this error anyways.
-            # pylint: disable=no-member
-            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
-                raise
-
-            logger.exception(
-                "failed to %s '%s' to '%s'", operation, from_info, to_info
-            )
-            return 1
-
-    return wrapper
-
-
-class LocalCache(CloudCache):
-    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-    CACHE_MODE = LocalTree.CACHE_MODE
-
-    def __init__(self, tree):
-        super().__init__(tree)
-        self.cache_dir = tree.config.get("url")
-
-    @property
-    def cache_dir(self):
-        return self.tree.path_info.fspath if self.tree.path_info else None
-
-    @cache_dir.setter
-    def cache_dir(self, value):
-        self.tree.path_info = PathInfo(value) if value else None
-
-    @classmethod
-    def supported(cls, config):  # pylint: disable=unused-argument
-        return True
-
-    @cached_property
-    def cache_path(self):
-        return os.path.abspath(self.cache_dir)
-
-    def hash_to_path(self, hash_):
-        # NOTE: `self.cache_path` is already normalized so we can simply use
-        # `os.sep` instead of `os.path.join`. This results in this helper
-        # being ~5.5 times faster.
-        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
-
-    def hashes_exist(
-        self, hashes, jobs=None, name=None
-    ):  # pylint: disable=unused-argument
-        return [
-            hash_
-            for hash_ in Tqdm(
-                hashes,
-                unit="file",
-                desc="Querying "
-                + ("cache in " + name if name else "local cache"),
-            )
-            if not self.changed_cache_file(hash_)
-        ]
-
-    def already_cached(self, path_info):
-        assert path_info.scheme in ["", "local"]
-
-        current_md5 = self.tree.get_hash(path_info)
-
-        if not current_md5:
-            return False
-
-        return not self.changed_cache(current_md5)
-
-    def _verify_link(self, path_info, link_type):
-        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
-            return
-
-        super()._verify_link(path_info, link_type)
-
-    @index_locked
-    def status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        # Return flattened dict containing all status info
-        dir_status, file_status, _ = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-        )
-        return dict(dir_status, **file_status)
-
-    def _status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
-
-        dir_status_info contains status for .dir files, file_status_info
-        contains status for all other files, and dir_contents is a dict of
-        {dir_hash: set(file_hash, ...)} which can be used to map
-        a .dir file to its file contents.
-        """
-        logger.debug(
-            f"Preparing to collect status from {remote.tree.path_info}"
-        )
-        md5s = set(named_cache.scheme_keys(self.tree.scheme))
-
-        logger.debug("Collecting information from local cache...")
-        local_exists = frozenset(
-            self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
-        )
-
-        # This is a performance optimization. We can safely assume that,
-        # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existence of
-        # those files.
-        if download and local_exists == md5s:
-            remote_exists = local_exists
-        else:
-            logger.debug("Collecting information from remote cache...")
-            remote_exists = set()
-            dir_md5s = set(named_cache.dir_keys(self.tree.scheme))
-            if dir_md5s:
-                remote_exists.update(
-                    self._indexed_dir_hashes(named_cache, remote, dir_md5s)
-                )
-                md5s.difference_update(remote_exists)
-            if md5s:
-                remote_exists.update(
-                    remote.hashes_exist(
-                        md5s, jobs=jobs, name=str(remote.tree.path_info)
-                    )
-                )
-        return self._make_status(
-            named_cache, show_checksums, local_exists, remote_exists
-        )
-
-    def _make_status(
-        self, named_cache, show_checksums, local_exists, remote_exists
-    ):
-        def make_names(hash_, names):
-            return {"name": hash_ if show_checksums else " ".join(names)}
-
-        dir_status = {}
-        file_status = {}
-        dir_contents = {}
-        for hash_, item in named_cache[self.tree.scheme].items():
-            if item.children:
-                dir_status[hash_] = make_names(hash_, item.names)
-                dir_contents[hash_] = set()
-                for child_hash, child in item.children.items():
-                    file_status[child_hash] = make_names(
-                        child_hash, child.names
-                    )
-                    dir_contents[hash_].add(child_hash)
-            else:
-                file_status[hash_] = make_names(hash_, item.names)
-
-        self._fill_statuses(dir_status, local_exists, remote_exists)
-        self._fill_statuses(file_status, local_exists, remote_exists)
-
-        self._log_missing_caches(dict(dir_status, **file_status))
-
-        return dir_status, file_status, dir_contents
-
-    def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
-        # Validate our index by verifying all indexed .dir hashes
-        # still exist on the remote
-        indexed_dirs = set(remote.index.dir_hashes())
-        indexed_dir_exists = set()
-        if indexed_dirs:
-            indexed_dir_exists.update(
-                remote.tree.list_hashes_exists(indexed_dirs)
-            )
-            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
-            if missing_dirs:
-                logger.debug(
-                    "Remote cache missing indexed .dir hashes '{}', "
-                    "clearing remote index".format(", ".join(missing_dirs))
-                )
-                remote.index.clear()
-
-        # Check if non-indexed (new) dir hashes exist on remote
-        dir_exists = dir_md5s.intersection(indexed_dir_exists)
-        dir_exists.update(
-            remote.tree.list_hashes_exists(dir_md5s - dir_exists)
-        )
-
-        # If .dir hash exists on the remote, assume directory contents
-        # still exists on the remote
-        for dir_hash in dir_exists:
-            file_hashes = list(
-                named_cache.child_keys(self.tree.scheme, dir_hash)
-            )
-            if dir_hash not in remote.index:
-                logger.debug(
-                    "Indexing new .dir '{}' with '{}' nested files".format(
-                        dir_hash, len(file_hashes)
-                    )
-                )
-                remote.index.update([dir_hash], file_hashes)
-            yield dir_hash
-            yield from file_hashes
-
-    @staticmethod
-    def _fill_statuses(hash_info_dir, local_exists, remote_exists):
-        # Using sets because they are way faster for lookups
-        local = set(local_exists)
-        remote = set(remote_exists)
-
-        for md5, info in hash_info_dir.items():
-            status = STATUS_MAP[(md5 in local, md5 in remote)]
-            info["status"] = status
-
-    def _get_plans(self, download, remote, status_info, status):
-        cache = []
-        path_infos = []
-        names = []
-        hashes = []
-        for md5, info in Tqdm(
-            status_info.items(), desc="Analysing status", unit="file"
-        ):
-            if info["status"] == status:
-                cache.append(self.tree.hash_to_path_info(md5))
-                path_infos.append(remote.tree.hash_to_path_info(md5))
-                names.append(info["name"])
-                hashes.append(md5)
-
-        if download:
-            to_infos = cache
-            from_infos = path_infos
-        else:
-            to_infos = path_infos
-            from_infos = cache
-
-        return from_infos, to_infos, names, hashes
-
-    def _process(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        logger.debug(
-            "Preparing to {} '{}'".format(
-                "download data from" if download else "upload data to",
-                remote.tree.path_info,
-            )
-        )
-
-        if download:
-            func = partial(
-                _log_exceptions(remote.tree.download, "download"),
-                dir_mode=self.tree.dir_mode,
-                file_mode=self.tree.file_mode,
-            )
-            status = STATUS_DELETED
-            desc = "Downloading"
-        else:
-            func = _log_exceptions(remote.tree.upload, "upload")
-            status = STATUS_NEW
-            desc = "Uploading"
-
-        if jobs is None:
-            jobs = remote.tree.JOBS
-
-        dir_status, file_status, dir_contents = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-        )
-
-        dir_plans = self._get_plans(download, remote, dir_status, status)
-        file_plans = self._get_plans(download, remote, file_status, status)
-
-        total = len(dir_plans[0]) + len(file_plans[0])
-        if total == 0:
-            return 0
-
-        with Tqdm(total=total, unit="file", desc=desc) as pbar:
-            func = pbar.wrap_fn(func)
-            with ThreadPoolExecutor(max_workers=jobs) as executor:
-                if download:
-                    from_infos, to_infos, names, _ = (
-                        d + f for d, f in zip(dir_plans, file_plans)
-                    )
-                    fails = sum(
-                        executor.map(func, from_infos, to_infos, names)
-                    )
-                else:
-                    # for uploads, push files first, and any .dir files last
-
-                    file_futures = {}
-                    for from_info, to_info, name, hash_ in zip(*file_plans):
-                        file_futures[hash_] = executor.submit(
-                            func, from_info, to_info, name
-                        )
-                    dir_futures = {}
-                    for from_info, to_info, name, dir_hash in zip(*dir_plans):
-                        wait_futures = {
-                            future
-                            for file_hash, future in file_futures.items()
-                            if file_hash in dir_contents[dir_hash]
-                        }
-                        dir_futures[dir_hash] = executor.submit(
-                            self._dir_upload,
-                            func,
-                            wait_futures,
-                            from_info,
-                            to_info,
-                            name,
-                        )
-                    fails = sum(
-                        future.result()
-                        for future in concat(
-                            file_futures.values(), dir_futures.values()
-                        )
-                    )
-
-        if fails:
-            if download:
-                remote.index.clear()
-                raise DownloadError(fails)
-            raise UploadError(fails)
-
-        if not download:
-            # index successfully pushed dirs
-            for dir_hash, future in dir_futures.items():
-                if future.result() == 0:
-                    file_hashes = dir_contents[dir_hash]
-                    logger.debug(
-                        "Indexing pushed dir '{}' with "
-                        "'{}' nested files".format(dir_hash, len(file_hashes))
-                    )
-                    remote.index.update([dir_hash], file_hashes)
-
-        return len(dir_plans[0]) + len(file_plans[0])
-
-    @staticmethod
-    def _dir_upload(func, futures, from_info, to_info, name):
-        for future in as_completed(futures):
-            if future.result():
-                # do not upload this .dir file if any file in this
-                # directory failed to upload
-                logger.debug(
-                    "failed to upload full contents of '{}', "
-                    "aborting .dir file upload".format(name)
-                )
-                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
-                return 1
-        return func(from_info, to_info, name)
-
-    @index_locked
-    def push(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=False,
-        )
-
-    @index_locked
-    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=True,
-        )
-
-    @staticmethod
-    def _log_missing_caches(hash_info_dict):
-        missing_caches = [
-            (md5, info)
-            for md5, info in hash_info_dict.items()
-            if info["status"] == STATUS_MISSING
-        ]
-        if missing_caches:
-            missing_desc = "\n".join(
-                "name: {}, md5: {}".format(info["name"], md5)
-                for md5, info in missing_caches
-            )
-            msg = (
-                "Some of the cache files do not exist neither locally "
-                "nor on remote. Missing cache files:\n{}".format(missing_desc)
-            )
-            logger.warning(msg)

dvc/checkout.py

@@ -0,0 +1,300 @@
+import logging
+
+from shortuuid import uuid
+
+import dvc.prompt as prompt
+from dvc.exceptions import (
+    CacheLinkError,
+    CheckoutError,
+    ConfirmRemoveError,
+    DvcException,
+)
+from dvc.objects import check, load
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.stage import stage
+from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
+    slow_link_guard,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _changed(path_info, fs, obj, cache):
+    logger.trace("checking if '%s'('%s') has changed.", path_info, obj)
+
+    try:
+        check(cache, obj)
+    except (FileNotFoundError, ObjectFormatError):
+        logger.debug(
+            "cache for '%s'('%s') has changed.", path_info, obj.hash_info
+        )
+        return True
+
+    try:
+        actual = stage(cache, path_info, fs, obj.hash_info.name).hash_info
+    except FileNotFoundError:
+        logger.debug("'%s' doesn't exist.", path_info)
+        return True
+
+    if obj.hash_info != actual:
+        logger.debug(
+            "hash value '%s' for '%s' has changed (actual '%s').",
+            obj.hash_info,
+            actual,
+            path_info,
+        )
+        return True
+
+    logger.trace("'%s' hasn't changed.", path_info)
+    return False
+
+
+def _remove(path_info, fs, cache, force=False):
+    if not fs.exists(path_info):
+        return
+
+    if force:
+        fs.remove(path_info)
+        return
+
+    current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM).hash_info
+    try:
+        obj = load(cache, current)
+        check(cache, obj)
+    except (FileNotFoundError, ObjectFormatError):
+        msg = (
+            f"file/directory '{path_info}' is going to be removed. "
+            "Are you sure you want to proceed?"
+        )
+
+        if not prompt.confirm(msg):
+            raise ConfirmRemoveError(str(path_info))
+
+    fs.remove(path_info)
+
+
+def _verify_link(cache, path_info, link_type):
+    if link_type == "hardlink" and cache.fs.getsize(path_info) == 0:
+        return
+
+    if cache.cache_type_confirmed:
+        return
+
+    is_link = getattr(cache.fs, f"is_{link_type}", None)
+    if is_link and not is_link(path_info):
+        cache.fs.remove(path_info)
+        raise DvcException(f"failed to verify {link_type}")
+
+    cache.cache_type_confirmed = True
+
+
+def _do_link(cache, from_info, to_info, link_method):
+    if cache.fs.exists(to_info):
+        raise DvcException(f"Link '{to_info}' already exists!")
+
+    link_method(from_info, to_info)
+
+    logger.debug(
+        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info,
+    )
+
+
+@slow_link_guard
+def _try_links(cache, from_info, to_info, link_types):
+    while link_types:
+        link_method = getattr(cache.fs, link_types[0])
+        try:
+            _do_link(cache, from_info, to_info, link_method)
+            _verify_link(cache, to_info, link_types[0])
+            return
+
+        except DvcException as exc:
+            logger.debug(
+                "Cache type '%s' is not supported: %s", link_types[0], exc
+            )
+            del link_types[0]
+
+    raise CacheLinkError([to_info])
+
+
+def _link(cache, from_info, to_info):
+    assert cache.fs.isfile(from_info)
+    cache.makedirs(to_info.parent)
+    _try_links(cache, from_info, to_info, cache.cache_types)
+
+
+def _cache_is_copy(cache, path_info):
+    """Checks whether cache uses copies."""
+    if cache.cache_type_confirmed:
+        return cache.cache_types[0] == "copy"
+
+    if set(cache.cache_types) <= {"copy"}:
+        return True
+
+    workspace_file = path_info.with_name("." + uuid())
+    test_cache_file = cache.fs.path_info / ".cache_type_test_file"
+    if not cache.fs.exists(test_cache_file):
+        cache.makedirs(test_cache_file.parent)
+        with cache.fs.open(test_cache_file, "wb") as fobj:
+            fobj.write(bytes(1))
+    try:
+        _link(cache, test_cache_file, workspace_file)
+    finally:
+        cache.fs.remove(workspace_file)
+        cache.fs.remove(test_cache_file)
+
+    cache.cache_type_confirmed = True
+    return cache.cache_types[0] == "copy"
+
+
+def _checkout_file(
+    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
+):
+    """The file is changed we need to checkout a new copy"""
+    modified = False
+    cache_info = cache.hash_to_path_info(obj.hash_info.value)
+    if fs.exists(path_info):
+        if not relink and _changed(path_info, fs, obj, cache):
+            modified = True
+            _remove(path_info, fs, cache, force=force)
+            _link(cache, cache_info, path_info)
+        else:
+            if fs.iscopy(path_info) and _cache_is_copy(cache, path_info):
+                cache.unprotect(path_info)
+            else:
+                _remove(path_info, fs, cache, force=force)
+                _link(cache, cache_info, path_info)
+    else:
+        _link(cache, cache_info, path_info)
+        modified = True
+
+    fs.repo.state.save(path_info, fs, obj.hash_info)
+    if progress_callback:
+        progress_callback(str(path_info))
+
+    return modified
+
+
+def _remove_redundant_files(path_info, fs, obj, cache, force):
+    existing_files = set(fs.walk_files(path_info))
+
+    needed_files = {path_info.joinpath(*key) for key, _ in obj}
+    redundant_files = existing_files - needed_files
+    for path in redundant_files:
+        _remove(path, fs, cache, force)
+
+    return bool(redundant_files)
+
+
+def _checkout_dir(
+    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
+):
+    modified = False, False
+    # Create dir separately so that dir is created
+    # even if there are no files in it
+    if not fs.exists(path_info):
+        modified = True
+        fs.makedirs(path_info)
+
+    logger.debug("Linking directory '%s'.", path_info)
+
+    for entry_key, entry_obj in obj:
+        entry_modified = _checkout_file(
+            path_info.joinpath(*entry_key),
+            fs,
+            entry_obj,
+            cache,
+            force,
+            progress_callback,
+            relink,
+        )
+        if entry_modified:
+            modified = True
+
+    modified = (
+        _remove_redundant_files(path_info, fs, obj, cache, force) or modified
+    )
+
+    fs.repo.state.save(path_info, fs, obj.hash_info)
+
+    # relink is not modified, assume it as nochange
+    return modified and not relink
+
+
+def _checkout(
+    path_info,
+    fs,
+    obj,
+    cache,
+    force=False,
+    progress_callback=None,
+    relink=False,
+):
+    if not obj.hash_info.isdir:
+        ret = _checkout_file(
+            path_info, fs, obj, cache, force, progress_callback, relink
+        )
+    else:
+        ret = _checkout_dir(
+            path_info, fs, obj, cache, force, progress_callback, relink,
+        )
+
+    fs.repo.state.save_link(path_info, fs)
+
+    return ret
+
+
+def checkout(
+    path_info,
+    fs,
+    obj,
+    cache,
+    force=False,
+    progress_callback=None,
+    relink=False,
+    quiet=False,
+):
+    if path_info.scheme not in ["local", cache.fs.scheme]:
+        raise NotImplementedError
+
+    failed = None
+    skip = False
+    if not obj:
+        if not quiet:
+            logger.warning(
+                "No file hash info found for '%s'. It won't be created.",
+                path_info,
+            )
+        _remove(path_info, fs, cache, force=force)
+        failed = path_info
+
+    elif not relink and not _changed(path_info, fs, obj, cache):
+        logger.trace("Data '%s' didn't change.", path_info)
+        skip = True
+    else:
+        try:
+            check(cache, obj)
+        except (FileNotFoundError, ObjectFormatError):
+            if not quiet:
+                logger.warning(
+                    "Cache '%s' not found. File '%s' won't be created.",
+                    obj.hash_info,
+                    path_info,
+                )
+            _remove(path_info, fs, cache, force=force)
+            failed = path_info
+
+    if failed or skip:
+        if progress_callback and obj:
+            progress_callback(
+                str(path_info), len(obj),
+            )
+        if failed:
+            raise CheckoutError([failed])
+        return
+
+    logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
+
+    return _checkout(
+        path_info, fs, obj, cache, force, progress_callback, relink,
+    )

dvc/cli.py

@@ -1,11 +1,14 @@
 """DVC command line interface"""
 import argparse
 import logging
+import os
 import sys
 
+from ._debug import add_debugging_flags
 from .command import (
     add,
     cache,
+    check_ignore,
     checkout,
     commit,
     completion,
@@ -25,6 +28,7 @@ from .command import (
     imp_url,
     init,
     install,
+    live,
     ls,
     metrics,
     move,
@@ -35,6 +39,7 @@ from .command import (
     repro,
     root,
     run,
+    stage,
     unprotect,
     update,
     version,
@@ -78,7 +83,10 @@ COMMANDS = [
     update,
     git_hook,
     plots,
+    stage,
     experiments,
+    check_ignore,
+    live,
 ]
 
 
@@ -134,7 +142,7 @@ def get_parent_parser():
     in order to prevent some weird behavior.
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
-
+    add_debugging_flags(parent_parser)
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."
@@ -181,6 +189,14 @@ def get_main_parser():
         help="Show program's version.",
     )
 
+    parser.add_argument(
+        "--cd",
+        default=os.path.curdir,
+        metavar="<path>",
+        help="Change to directory before executing.",
+        type=str,
+    )
+
     # Sub commands
     subparsers = parser.add_subparsers(
         title="Available Commands",

dvc/command/add.py

@@ -3,13 +3,17 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
 
 
 class CmdAdd(CmdBase):
     def run(self):
+        from dvc.exceptions import (
+            DvcException,
+            RecursiveAddingWhileUsingFilename,
+        )
+
         try:
             if len(self.args.targets) > 1 and self.args.file:
                 raise RecursiveAddingWhileUsingFilename()
@@ -20,6 +24,12 @@ class CmdAdd(CmdBase):
                 no_commit=self.args.no_commit,
                 fname=self.args.file,
                 external=self.args.external,
+                glob=self.args.glob,
+                desc=self.args.desc,
+                out=self.args.out,
+                remote=self.args.remote,
+                to_remote=self.args.to_remote,
+                jobs=self.args.jobs,
             )
 
         except DvcException:
@@ -57,11 +67,55 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Allow targets that are outside of the DVC repository.",
     )
+    parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Allows targets containing shell-style wildcards.",
+    )
     parser.add_argument(
         "--file",
-        help="Specify name of the DVC-file this command will generate.",
+        help="Specify name of the .dvc file this command will generate.",
         metavar="<filename>",
     )
+    parser.add_argument(
+        "-o",
+        "--out",
+        help="Destination path to put files to.",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--to-remote",
+        action="store_true",
+        default=False,
+        help="Download it directly to the remote",
+    )
+    parser.add_argument(
+        "-r",
+        "--remote",
+        help="Remote storage to download to",
+        metavar="<name>",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
+    parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
     parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add.",
     ).complete = completion.FILE

dvc/command/base.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from abc import ABC, abstractmethod
 
 logger = logging.getLogger(__name__)
@@ -28,24 +29,21 @@ def append_doc_link(help_message, path):
 
 
 class CmdBase(ABC):
+    UNINITIALIZED = False
+
     def __init__(self, args):
         from dvc.repo import Repo
         from dvc.updater import Updater
 
-        self.repo = Repo()
+        os.chdir(args.cd)
+
+        self.repo = Repo(uninitialized=self.UNINITIALIZED)
         self.config = self.repo.config
         self.args = args
-        hardlink_lock = self.config["core"].get("hardlink_lock", False)
-        updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
-        updater.check()
-
-    @property
-    def default_targets(self):
-        """Default targets for `dvc repro`."""
-        from dvc.dvcfile import PIPELINE_FILE
-
-        logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
-        return [PIPELINE_FILE]
+        if self.repo.tmp_dir:
+            hardlink_lock = self.config["core"].get("hardlink_lock", False)
+            updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
+            updater.check()
 
     @abstractmethod
     def run(self):
@@ -55,3 +53,5 @@ class CmdBase(ABC):
 class CmdBaseNoRepo(CmdBase):
     def __init__(self, args):  # pylint: disable=super-init-not-called
         self.args = args
+
+        os.chdir(args.cd)

dvc/command/cache.py

@@ -1,14 +1,31 @@
 import argparse
+import logging
 
 from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
+logger = logging.getLogger(__name__)
+
 
 class CmdCacheDir(CmdConfig):
     def run(self):
-        with self.config.edit(level=self.args.level) as edit:
-            edit["cache"]["dir"] = self.args.value
+        if self.args.value is None and not self.args.unset:
+            if self.args.level:
+                conf = self.config.read(level=self.args.level)
+            else:
+                # Use merged config with default values
+                conf = self.config
+            self._check(conf, False, "cache", "dir")
+            logger.info(conf["cache"]["dir"])
+            return 0
+        with self.config.edit(level=self.args.level) as conf:
+            if self.args.unset:
+                self._check(conf, False, "cache", "dir")
+                del conf["cache"]["dir"]
+            else:
+                self._check(conf, False, "cache")
+                conf["cache"]["dir"] = self.args.value
         return 0
 
 
@@ -55,6 +72,8 @@ def add_parser(subparsers, parent_parser):
         "value",
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
-        "config file location.",
+        "config file location. If no path is provided, it returns the "
+        "current cache directory.",
+        nargs="?",
     ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/check_ignore.py

@@ -0,0 +1,127 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCheckIgnore(CmdBase):
+    def __init__(self, args):
+        super().__init__(args)
+        self.ignore_filter = self.repo.fs.dvcignore
+
+    def _show_results(self, result):
+        if not result.match and not self.args.non_matching:
+            return
+
+        if self.args.details:
+            patterns = result.patterns
+            if not self.args.all:
+                patterns = patterns[-1:]
+
+            for pattern in patterns:
+                logger.info("{}\t{}".format(pattern, result.file))
+        else:
+            logger.info(result.file)
+
+    def _check_one_file(self, target):
+        result = self.ignore_filter.check_ignore(target)
+        self._show_results(result)
+        if result.match:
+            return 0
+        return 1
+
+    def _interactive_mode(self):
+        ret = 1
+        while True:
+            try:
+                target = input()
+            except (KeyboardInterrupt, EOFError):
+                break
+            if target == "":
+                break
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _normal_mode(self):
+        ret = 1
+        for target in self.args.targets:
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _check_args(self):
+        from dvc.exceptions import DvcException
+
+        if not self.args.stdin and not self.args.targets:
+            raise DvcException("`targets` or `--stdin` needed")
+
+        if self.args.stdin and self.args.targets:
+            raise DvcException("cannot have both `targets` and `--stdin`")
+
+        if self.args.non_matching and not self.args.details:
+            raise DvcException(
+                "`--non-matching` is only valid with `--details`"
+            )
+
+        if self.args.all and not self.args.details:
+            raise DvcException("`--all` is only valid with `--details`")
+
+        if self.args.quiet and self.args.details:
+            raise DvcException("cannot use both `--details` and `--quiet`")
+
+    def run(self):
+        self._check_args()
+        if self.args.stdin:
+            return self._interactive_mode()
+        return self._normal_mode()
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = (
+        "Check whether files or directories are excluded due to `.dvcignore`."
+    )
+
+    parser = subparsers.add_parser(
+        "check-ignore",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "check-ignore"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-d",
+        "--details",
+        action="store_true",
+        default=False,
+        help="Show the exclude patterns along with each target path.",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "-n",
+        "--non-matching",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "--stdin",
+        action="store_true",
+        default=False,
+        help="Read paths from standard input instead of providing `targets`.",
+    )
+    parser.add_argument(
+        "targets", nargs="*", help="File or directory paths to check",
+    ).complete = completion.FILE
+    parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -7,7 +7,6 @@ import colorama
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
-from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
@@ -39,6 +38,8 @@ def log_changes(stats):
 
 class CmdCheckout(CmdBase):
     def run(self):
+        from dvc.utils.humanize import get_summary
+
         stats, exc = None, None
         try:
             stats = self.repo.checkout(

dvc/command/commit.py

@@ -3,13 +3,14 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdCommit(CmdBase):
     def run(self):
+        from dvc.exceptions import DvcException
+
         if not self.args.targets:
             self.args.targets = [None]
 
@@ -32,7 +33,10 @@ class CmdCommit(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = "Save changed data to cache and update DVC-files."
+    COMMIT_HELP = (
+        "Record changes to files or directories tracked by DVC"
+        " by storing the current versions in the cache."
+    )
 
     commit_parser = subparsers.add_parser(
         "commit",
@@ -65,7 +69,7 @@ def add_parser(subparsers, parent_parser):
     commit_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-files to commit. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
+        help="stages or .dvc files to commit. Optional. "
+        "(Finds all DVC files in the workspace by default.)",
     ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -8,10 +8,7 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link
 logger = logging.getLogger(__name__)
 FILE = shtab.FILE
 DIR = shtab.DIRECTORY
-DVC_FILE = {
-    "bash": "_dvc_compgen_DVCFiles",
-    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
-}
+
 PREAMBLE = {
     "bash": """
 # $1=COMP_WORDS[1]
@@ -21,8 +18,73 @@ _dvc_compgen_DVCFiles() {
   compgen -f -X '!*Dvcfile' -- $1
   compgen -f -X '!*dvc.yaml' -- $1
 }
-""",
-    "zsh": "",
+
+_dvc_compgen_stages() {
+    local _dvc_stages=($(dvc stage list -q --names-only))
+    compgen -W "${_dvc_stages[*]}" -- $1
+}
+_dvc_compgen_stages_and_files() {
+    _dvc_compgen_DVCFiles $1
+    _dvc_compgen_stages $1
+}
+
+_dvc_compgen_exps() {
+    local _dvc_exps=($(dvc exp list -q --all --names-only))
+    compgen -W "${_dvc_exps[*]}" -- $1
+}
+    """,
+    "zsh": """
+_dvc_compadd_DVCFiles() {
+    _files -g '(*?.dvc|Dvcfile|dvc.yaml)'
+}
+_dvc_compadd_stages() {
+    # this will also show up the description of the stages
+    _describe 'stages' "($(_dvc_stages_output))"
+}
+
+_dvc_stages_output() {
+  dvc stage list -q | awk '{
+    # escape possible `:` on the stage name
+    sub(/:/, "\\\\\\\\:", $1);
+    # read all of the columns except the first
+    # reading `out` from $2, so as not to have a leading whitespace
+    out=$2; for(i=3;i<=NF;i++){out=out" "$i};
+    # print key, ":" and then single-quote the description
+    # colon is a delimiter used by `_describe` to separate field/description
+    print $1":""\\047"out"\\047"
+    # single quote -> \\047
+    }'
+}
+
+_dvc_compadd_stages_and_files() {
+    _dvc_compadd_DVCFiles
+    _dvc_compadd_stages
+}
+
+_dvc_compadd_exps() {
+    _describe 'experiments' "($(dvc exp list -q -a --names-only))"
+}
+    """,
+}
+
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_dvc_compadd_DVCFiles",
+}
+
+STAGE = {
+    "bash": "_dvc_compgen_stages",
+    "zsh": "_dvc_compadd_stages",
+}
+
+DVCFILES_AND_STAGE = {
+    "bash": "_dvc_compgen_stages_and_files",
+    "zsh": "_dvc_compadd_stages_and_files",
+}
+
+EXPERIMENT = {
+    "bash": "_dvc_compgen_exps",
+    "zsh": "_dvc_compadd_exps",
 }
 
 

dvc/command/config.py

@@ -1,33 +1,115 @@
 import argparse
 import logging
+import os
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.config import Config, ConfigError
 
 logger = logging.getLogger(__name__)
 
+NAME_REGEX = r"^(?P<remote>remote\.)?(?P<section>[^\.]*)\.(?P<option>[^\.]*)$"
+
+
+def _name_type(value):
+    import re
+
+    match = re.match(NAME_REGEX, value)
+    if not match:
+        raise argparse.ArgumentTypeError(
+            "name argument should look like "
+            "remote.name.option or "
+            "section.option"
+        )
+    return (
+        bool(match.group("remote")),
+        match.group("section").lower(),
+        match.group("option").lower(),
+    )
+
 
 class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
+        from dvc.config import Config
+
         super().__init__(args)
 
         self.config = Config(validate=False)
 
     def run(self):
-        section, opt = self.args.name.lower().strip().split(".", 1)
+        if self.args.show_origin:
+            if any((self.args.value, self.args.unset)):
+                logger.error(
+                    "--show-origin can't be used together with any of these "
+                    "options: -u/--unset, value"
+                )
+                return 1
+
+        if self.args.list:
+            return self._list()
+
+        if self.args.name is None:
+            logger.error("name argument is required")
+            return 1
+
+        remote, section, opt = self.args.name
 
         if self.args.value is None and not self.args.unset:
-            conf = self.config.load_one(self.args.level)
-            self._check(conf, section, opt)
-            logger.info(conf[section][opt])
-            return 0
+            return self._get(remote, section, opt)
 
+        return self._set(remote, section, opt)
+
+    def _list(self):
+        if any((self.args.name, self.args.value, self.args.unset)):
+            logger.error(
+                "-l/--list can't be used together with any of these "
+                "options: -u/--unset, name, value"
+            )
+            return 1
+
+        levels = [self.args.level] if self.args.level else self.config.LEVELS
+        for level in levels:
+            conf = self.config.read(level)
+            prefix = self._config_file_prefix(
+                self.args.show_origin, self.config, level
+            )
+            logger.info("\n".join(self._format_config(conf, prefix)))
+
+        return 0
+
+    def _get(self, remote, section, opt):
+        from dvc.config import ConfigError
+
+        levels = (
+            [self.args.level] if self.args.level else self.config.LEVELS[::-1]
+        )
+
+        for level in levels:
+            conf = self.config.read(level)
+            if remote:
+                conf = conf["remote"]
+
+            try:
+                self._check(conf, remote, section, opt)
+            except ConfigError:
+                if self.args.level:
+                    raise
+            else:
+                prefix = self._config_file_prefix(
+                    self.args.show_origin, self.config, level
+                )
+                logger.info("{}{}".format(prefix, conf[section][opt]))
+                break
+
+        return 0
+
+    def _set(self, remote, section, opt):
         with self.config.edit(self.args.level) as conf:
+            if remote:
+                conf = conf["remote"]
             if self.args.unset:
-                self._check(conf, section, opt)
+                self._check(conf, remote, section, opt)
                 del conf[section][opt]
             else:
-                self._check(conf, section)
+                self._check(conf, remote, section)
                 conf[section][opt] = self.args.value
 
         if self.args.name == "cache.type":
@@ -39,14 +121,39 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
-    def _check(self, conf, section, opt=None):
+    def _check(self, conf, remote, section, opt=None):
+        from dvc.config import ConfigError
+
+        name = "remote" if remote else "section"
         if section not in conf:
-            msg = "section {} doesn't exist"
-            raise ConfigError(msg.format(self.args.name))
+            raise ConfigError(f"{name} '{section}' doesn't exist")
 
         if opt and opt not in conf[section]:
-            msg = "option {} doesn't exist"
-            raise ConfigError(msg.format(self.args.name))
+            raise ConfigError(
+                f"option '{opt}' doesn't exist in {name} '{section}'"
+            )
+
+    @staticmethod
+    def _format_config(config, prefix=""):
+        from dvc.utils.flatten import flatten
+
+        for key, value in flatten(config).items():
+            yield f"{prefix}{key}={value}"
+
+    @staticmethod
+    def _config_file_prefix(show_origin, config, level):
+        from dvc.repo import Repo
+
+        if not show_origin:
+            return ""
+
+        level = level or "repo"
+        fname = config.files[level]
+
+        if level in ["local", "repo"]:
+            fname = os.path.relpath(fname, start=Repo.find_root())
+
+        return fname + "\t"
 
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
@@ -65,14 +172,21 @@ level_group.add_argument(
     const="system",
     help="Use system config.",
 )
+level_group.add_argument(
+    "--project",
+    dest="level",
+    action="store_const",
+    const="repo",
+    help="Use project config (.dvc/config).",
+)
 level_group.add_argument(
     "--local",
     dest="level",
     action="store_const",
     const="local",
-    help="Use local config.",
+    help="Use local config (.dvc/config.local).",
 )
-parent_config_parser.set_defaults(level="repo")
+parent_config_parser.set_defaults(level=None)
 
 
 def add_parser(subparsers, parent_parser):
@@ -92,6 +206,24 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Unset option.",
     )
-    config_parser.add_argument("name", help="Option name.")
+    config_parser.add_argument(
+        "name",
+        nargs="?",
+        type=_name_type,
+        help="Option name (section.option or remote.name.option).",
+    )
     config_parser.add_argument("value", nargs="?", help="Option value.")
+    config_parser.add_argument(
+        "-l",
+        "--list",
+        default=False,
+        action="store_true",
+        help="List all defined config values.",
+    )
+    config_parser.add_argument(
+        "--show-origin",
+        default=False,
+        action="store_true",
+        help="Show the source file containing each config value.",
+    )
     config_parser.set_defaults(func=CmdConfig)

dvc/command/daemon.py

@@ -9,8 +9,9 @@ class CmdDaemonBase(CmdBaseNoRepo):
 class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
-        from dvc.repo import Repo
+
         from dvc.config import Config
+        from dvc.repo import Repo
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()

dvc/command/dag.py

@@ -2,14 +2,13 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 def _show_ascii(G):
-    from dvc.repo.graph import get_pipelines
     from dvc.dagascii import draw
+    from dvc.repo.graph import get_pipelines
 
     pipelines = get_pipelines(G)
 
@@ -22,6 +21,7 @@ def _show_ascii(G):
 
 def _show_dot(G):
     import io
+
     from networkx.drawing.nx_pydot import write_dot
 
     dot_file = io.StringIO()
@@ -29,57 +29,85 @@ def _show_dot(G):
     return dot_file.getvalue()
 
 
-def _build(G, target=None, full=False):
+def _collect_targets(repo, target, outs):
+    if not target:
+        return []
+
+    pairs = repo.stage.collect_granular(target)
+    if not outs:
+        return [stage.addressing for stage, _ in pairs]
+
+    targets = []
+    for stage, info in pairs:
+        if not info:
+            targets.extend([str(out) for out in stage.outs])
+            continue
+
+        for out in repo.outs_trie.itervalues(prefix=info.parts):  # noqa: B301
+            targets.extend(str(out))
+
+    return targets
+
+
+def _transform(repo, outs):
     import networkx as nx
-    from dvc.repo.graph import get_pipeline, get_pipelines
 
-    if target:
-        H = get_pipeline(get_pipelines(G), target)
-        if not full:
-            descendants = nx.descendants(G, target)
+    from dvc.stage import Stage
+
+    def _relabel(node) -> str:
+        return node.addressing if isinstance(node, Stage) else str(node)
+
+    G = repo.outs_graph if outs else repo.graph
+    return nx.relabel_nodes(G, _relabel, copy=True)
+
+
+def _filter(G, targets, full):
+    import networkx as nx
+
+    if not targets:
+        return G
+
+    H = G.copy()
+    if not full:
+        descendants = set()
+        for target in targets:
+            descendants.update(nx.descendants(G, target))
             descendants.add(target)
-            H.remove_nodes_from(set(G.nodes()) - descendants)
-    else:
-        H = G
+        H.remove_nodes_from(set(G.nodes()) - descendants)
 
-    def _relabel(stage):
-        return stage.addressing
+    undirected = H.to_undirected()
+    connected = set()
+    for target in targets:
+        connected.update(nx.node_connected_component(undirected, target))
 
-    return nx.relabel_nodes(H, _relabel, copy=False)
+    H.remove_nodes_from(set(H.nodes()) - connected)
+
+    return H
+
+
+def _build(repo, target=None, full=False, outs=False):
+    targets = _collect_targets(repo, target, outs)
+    G = _transform(repo, outs)
+    return _filter(G, targets, full)
 
 
 class CmdDAG(CmdBase):
     def run(self):
-        try:
-            target = None
-            if self.args.target:
-                stages = self.repo.collect(self.args.target)
-                if len(stages) > 1:
-                    logger.error(
-                        f"'{self.args.target}' contains more than one stage "
-                        "{stages}, please specify one stage"
-                    )
-                    return 1
-                target = stages[0]
+        G = _build(
+            self.repo,
+            target=self.args.target,
+            full=self.args.full,
+            outs=self.args.outs,
+        )
 
-            G = _build(self.repo.graph, target=target, full=self.args.full,)
+        if self.args.dot:
+            logger.info(_show_dot(G))
+        else:
+            from dvc.utils.pager import pager
 
-            if self.args.dot:
-                logger.info(_show_dot(G))
-            else:
-                from dvc.utils.pager import pager
+            pager(_show_ascii(G))
 
-                pager(_show_ascii(G))
-
-            return 0
-        except DvcException:
-            msg = "failed to show "
-            if self.args.target:
-                msg += f"a pipeline for '{target}'"
-            else:
-                msg += "pipelines"
-            logger.exception(msg)
-            return 1
+        return 0
 
 
 def add_parser(subparsers, parent_parser):
@@ -106,6 +134,13 @@ def add_parser(subparsers, parent_parser):
             "showing DAG consisting only of ancestors."
         ),
     )
+    dag_parser.add_argument(
+        "-o",
+        "--outs",
+        action="store_true",
+        default=False,
+        help="Print output files instead of stages.",
+    )
     dag_parser.add_argument(
         "target",
         nargs="?",

dvc/command/data_sync.py

@@ -3,25 +3,28 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.checkout import log_changes
-from dvc.exceptions import CheckoutError, DvcException
-from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
     def log_summary(self, stats):
+        from dvc.utils.humanize import get_summary
+
         default_msg = "Everything is up to date."
         logger.info(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
     def log_summary(self, stats):
+        from dvc.command.checkout import log_changes
+
         log_changes(stats)
         super().log_summary(stats)
 
     def run(self):
+        from dvc.exceptions import CheckoutError, DvcException
+
         try:
             stats = self.repo.pull(
                 targets=self.args.targets,
@@ -34,6 +37,7 @@ class CmdDataPull(CmdDataBase):
                 force=self.args.force,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
+                glob=self.args.glob,
             )
             self.log_summary(stats)
         except (CheckoutError, DvcException) as exc:
@@ -46,6 +50,8 @@ class CmdDataPull(CmdDataBase):
 
 class CmdDataPush(CmdDataBase):
     def run(self):
+        from dvc.exceptions import DvcException
+
         try:
             processed_files_count = self.repo.push(
                 targets=self.args.targets,
@@ -57,6 +63,7 @@ class CmdDataPush(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
+                glob=self.args.glob,
             )
             self.log_summary({"pushed": processed_files_count})
         except DvcException:
@@ -67,6 +74,8 @@ class CmdDataPush(CmdDataBase):
 
 class CmdDataFetch(CmdDataBase):
     def run(self):
+        from dvc.exceptions import DvcException
+
         try:
             processed_files_count = self.repo.fetch(
                 targets=self.args.targets,
@@ -97,7 +106,11 @@ def shared_parent_parser():
         "-j",
         "--jobs",
         type=int,
-        help="Number of jobs to run simultaneously.",
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
         metavar="<number>",
     )
     parent_parser.add_argument(
@@ -175,6 +188,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch run history for all stages.",
     )
+    pull_parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Pull cache for targets matching shell-style wildcards.",
+    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
@@ -230,12 +249,17 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Push run history for all stages.",
     )
+    push_parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Allows targets containing shell-style wildcards.",
+    )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
     FETCH_HELP = (
-        "Get tracked files or directories from remote storage"
-        " into the cache."
+        "Download files or directories from remote storage to the cache."
     )
 
     fetch_parser = subparsers.add_parser(

dvc/command/destroy.py

@@ -3,13 +3,14 @@ import logging
 
 import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
     def run(self):
+        from dvc.exceptions import DvcException
+
         try:
             statement = (
                 "This will destroy all information about your pipelines,"
@@ -25,14 +26,14 @@ class CmdDestroy(CmdBase):
                 )
 
             self.repo.destroy()
-        except Exception:  # noqa, pylint: disable=broad-except
+        except DvcException:
             logger.exception("failed to destroy DVC")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = "Remove DVC-files, local DVC config and data cache."
+    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
 
     destroy_parser = subparsers.add_parser(
         "destroy",

dvc/command/diff.py

@@ -5,36 +5,51 @@ import os
 
 import colorama
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
-def _show_md(diff):
+def _digest(checksum):
+    if isinstance(checksum, str):
+        return checksum[0:8]
+    return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+
+def _show_md(diff, show_hash=False, hide_missing=False):
     from dvc.utils.diff import table
 
+    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
-    for status in ["added", "deleted", "modified"]:
+    statuses = ["added", "deleted", "renamed", "modified"]
+    if not hide_missing:
+        statuses.append("not in cache")
+    for status in statuses:
         entries = diff.get(status, [])
         if not entries:
             continue
-        paths = sorted(entry["path"] for entry in entries)
-        for path in paths:
-            rows.append([status, path])
+        for entry in entries:
+            path = entry["path"]
+            if isinstance(path, dict):
+                path = f"{path['old']} -> {path['new']}"
+            if show_hash:
+                check_sum = _digest(entry.get("hash", ""))
+                rows.append([status, check_sum, path])
+            else:
+                rows.append([status, path])
 
-    return table(["Status", "Path"], rows, True)
+    return table(header, rows, True)
 
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _format(diff):
+    def _format(diff, hide_missing=False):
         """
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
 
-        A group's header is colored and its entries are sorted to enhance
-        readability, for example:
+        A group's header is colored to enhance readability, for example:
 
             Added:
                 another_file.txt
@@ -55,21 +70,21 @@ class CmdDiff(CmdBase):
         At the bottom, include a summary with the number of files per state.
         """
 
-        def _digest(checksum):
-            if isinstance(checksum, str):
-                return checksum[0:8]
-            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
-
         colors = {
             "added": colorama.Fore.GREEN,
             "modified": colorama.Fore.YELLOW,
             "deleted": colorama.Fore.RED,
+            "renamed": colorama.Fore.GREEN,
+            "not in cache": colorama.Fore.YELLOW,
         }
 
         summary = {}
         groups = []
 
-        for state in ["added", "deleted", "modified"]:
+        states = ["added", "deleted", "renamed", "modified"]
+        if not hide_missing:
+            states.append("not in cache")
+        for state in states:
             summary[state] = 0
             entries = diff[state]
 
@@ -80,6 +95,8 @@ class CmdDiff(CmdBase):
 
             for entry in entries:
                 path = entry["path"]
+                if isinstance(path, dict):
+                    path = f"{path['old']} -> {path['new']}"
                 checksum = entry.get("hash")
                 summary[state] += 1 if not path.endswith(os.sep) else 0
                 content.append(
@@ -87,7 +104,7 @@ class CmdDiff(CmdBase):
                         space="    ",
                         checksum=_digest(checksum) if checksum else "",
                         separator="  " if checksum else "",
-                        path=entry["path"],
+                        path=path,
                     )
                 )
 
@@ -100,28 +117,51 @@ class CmdDiff(CmdBase):
                 )
             )
 
-        groups.append(
+        if not sum(summary.values()):
+            return None
+
+        fmt = (
             "files summary: {added} added, {deleted} deleted,"
-            " {modified} modified".format_map(summary)
+            " {renamed} renamed, {modified} modified"
         )
+        if not hide_missing:
+            fmt += ", {not in cache} not in cache"
+        groups.append(fmt.format_map(summary))
 
         return "\n\n".join(groups)
 
     def run(self):
-        try:
-            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+        from dvc.exceptions import DvcException
 
-            if not self.args.show_hash:
-                for _, entries in diff.items():
+        try:
+            diff = self.repo.diff(
+                self.args.a_rev, self.args.b_rev, self.args.targets
+            )
+            show_hash = self.args.show_hash
+            hide_missing = self.args.b_rev or self.args.hide_missing
+            if hide_missing:
+                del diff["not in cache"]
+
+            for key, entries in diff.items():
+                entries = sorted(
+                    entries,
+                    key=lambda entry: entry["path"]["old"]
+                    if isinstance(entry["path"], dict)
+                    else entry["path"],
+                )
+                if not show_hash:
                     for entry in entries:
                         del entry["hash"]
+                diff[key] = entries
 
             if self.args.show_json:
                 logger.info(json.dumps(diff))
             elif self.args.show_md:
-                logger.info(_show_md(diff))
+                logger.info(_show_md(diff, show_hash, hide_missing))
             elif diff:
-                logger.info(self._format(diff))
+                output = self._format(diff, hide_missing)
+                if output:
+                    logger.info(output)
 
         except DvcException:
             logger.exception("failed to get diff")
@@ -141,6 +181,15 @@ def add_parser(subparsers, parent_parser):
         help=DIFF_DESCRIPTION,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help=(
+            "Specific DVC-tracked files to compare. "
+            "Accepts one or more file paths."
+        ),
+        metavar="<paths>",
+    ).complete = completion.FILE
     diff_parser.add_argument(
         "a_rev",
         help="Old Git commit to compare (defaults to HEAD)",
@@ -170,4 +219,9 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
     )
+    diff_parser.add_argument(
+        "--hide-missing",
+        help="Hide missing cache file status.",
+        action="store_true",
+    )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/gc.py

@@ -30,6 +30,8 @@ class CmdGC(CmdBase):
             msg += " and all git branches"
         elif self.args.all_tags:
             msg += " and all git tags"
+        elif self.args.all_experiments:
+            msg += " and all experiments"
 
         if self.args.repos:
             msg += " of the current and the following repos:"
@@ -49,6 +51,7 @@ class CmdGC(CmdBase):
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
             all_commits=self.args.all_commits,
+            all_experiments=self.args.all_experiments,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -99,6 +102,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Keep data files for all Git commits.",
     )
+    gc_parser.add_argument(
+        "--all-experiments",
+        action="store_true",
+        default=False,
+        help="Keep data files for all experiments.",
+    )
     gc_parser.add_argument(
         "-c",
         "--cloud",
@@ -123,7 +132,11 @@ def add_parser(subparsers, parent_parser):
         "-j",
         "--jobs",
         type=int,
-        help="Number of jobs to run simultaneously.",
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
         metavar="<number>",
     )
     gc_parser.add_argument(

dvc/command/get.py

@@ -39,6 +39,7 @@ class CmdGet(CmdBaseNoRepo):
                 path=self.args.path,
                 out=self.args.out,
                 rev=self.args.rev,
+                jobs=self.args.jobs,
             )
             return 0
         except DvcException:
@@ -84,4 +85,15 @@ def add_parser(subparsers, parent_parser):
         help="Print the storage location (URL) the target data would be "
         "downloaded from, and exit.",
     )
+    get_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -14,7 +14,7 @@ class CmdGetUrl(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            Repo.get_url(self.args.url, out=self.args.out)
+            Repo.get_url(self.args.url, out=self.args.out, jobs=self.args.jobs)
             return 0
         except DvcException:
             logger.exception(f"failed to get '{self.args.url}'")
@@ -36,4 +36,15 @@ def add_parser(subparsers, parent_parser):
     get_parser.add_argument(
         "out", nargs="?", help="Destination path to put data to.",
     ).complete = completion.DIR
+    get_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -61,6 +61,25 @@ class CmdPrePush(CmdHookBase):
         return main(["push"])
 
 
+class CmdMergeDriver(CmdHookBase):
+    def _run(self):
+        from dvc.dvcfile import Dvcfile
+        from dvc.repo import Repo
+
+        dvc = Repo()
+
+        try:
+            ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
+            our = Dvcfile(dvc, self.args.our, verify=False)
+            their = Dvcfile(dvc, self.args.their, verify=False)
+
+            our.merge(ancestor, their)
+
+            return 0
+        finally:
+            dvc.close()
+
+
 def add_parser(subparsers, parent_parser):
     GIT_HOOK_HELP = "Run GIT hook."
 
@@ -113,3 +132,27 @@ def add_parser(subparsers, parent_parser):
         "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
+
+    MERGE_DRIVER_HELP = "Run GIT merge driver."
+    merge_driver_parser = git_hook_subparsers.add_parser(
+        "merge-driver",
+        parents=[parent_parser],
+        description=MERGE_DRIVER_HELP,
+        help=MERGE_DRIVER_HELP,
+    )
+    merge_driver_parser.add_argument(
+        "--ancestor",
+        required=True,
+        help="Ancestor's version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--our",
+        required=True,
+        help="Current version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--their",
+        required=True,
+        help="Other branch's version of the conflicting file.",
+    )
+    merge_driver_parser.set_defaults(func=CmdMergeDriver)

dvc/command/imp.py

@@ -15,7 +15,11 @@ class CmdImport(CmdBase):
                 self.args.url,
                 self.args.path,
                 out=self.args.out,
+                fname=self.args.file,
                 rev=self.args.rev,
+                no_exec=self.args.no_exec,
+                desc=self.args.desc,
+                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception(
@@ -59,4 +63,35 @@ def add_parser(subparsers, parent_parser):
         help="Git revision (e.g. SHA, branch, tag)",
         metavar="<commit>",
     )
+    import_parser.add_argument(
+        "--file",
+        help="Specify name of the .dvc file this command will generate.",
+        metavar="<filename>",
+    )
+    import_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create .dvc file without actually downloading it.",
+    )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    import_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -16,6 +16,10 @@ class CmdImportUrl(CmdBase):
                 out=self.args.out,
                 fname=self.args.file,
                 no_exec=self.args.no_exec,
+                remote=self.args.remote,
+                to_remote=self.args.to_remote,
+                desc=self.args.desc,
+                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception(
@@ -42,33 +46,61 @@ def add_parser(subparsers, parent_parser):
     )
     import_parser.add_argument(
         "url",
-        help="Supported urls:\n"
-        "/path/to/file\n"
-        "/path/to/directory\n"
-        "C:\\\\path\\to\\file\n"
-        "C:\\\\path\\to\\directory\n"
+        help="Location of the data to download. Supported URLs:\n"
+        "/absolute/path/to/file/or/dir\n"
+        "relative/path/to/file/or/dir\n"
+        "C:\\\\path\\to\\file\\or\\dir\n"
         "https://example.com/path/to/file\n"
-        "s3://bucket/path/to/file\n"
-        "s3://bucket/path/to/directory\n"
-        "gs://bucket/path/to/file\n"
-        "gs://bucket/path/to/directory\n"
+        "s3://bucket/key/path\n"
+        "gs://bucket/path/to/file/or/dir\n"
         "hdfs://example.com/path/to/file\n"
-        "ssh://example.com:/path/to/file\n"
-        "ssh://example.com:/path/to/directory\n"
-        "remote://myremote/path/to/file (see `dvc remote`)",
+        "ssh://example.com/absolute/path/to/file/or/dir\n"
+        "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
         "out", nargs="?", help="Destination path to put files to.",
     ).complete = completion.DIR
     import_parser.add_argument(
         "--file",
-        help="Specify name of the DVC-file this command will generate.",
+        help="Specify name of the .dvc file this command will generate.",
         metavar="<filename>",
     ).complete = completion.DIR
     import_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create DVC-file without actually downloading it.",
+        help="Only create .dvc file without actually downloading it.",
+    )
+    import_parser.add_argument(
+        "--to-remote",
+        action="store_true",
+        default=False,
+        help="Download it directly to the remote",
+    )
+    import_parser.add_argument(
+        "-r",
+        "--remote",
+        help="Remote storage to download to",
+        metavar="<name>",
+    )
+    import_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
     )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/init.py

@@ -1,15 +1,42 @@
 import argparse
 import logging
 
+import colorama
+
+from dvc import analytics
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.utils import boxify
+from dvc.utils import format_link as fmt_link
 
 logger = logging.getLogger(__name__)
 
 
+def _welcome_message():
+    if analytics.is_enabled():
+        logger.info(
+            boxify(
+                "DVC has enabled anonymous aggregate usage analytics.\n"
+                "Read the analytics documentation (and how to opt-out) here:\n"
+                + fmt_link("https://dvc.org/doc/user-guide/analytics"),
+                border_color="red",
+            )
+        )
+
+    msg = (
+        "{yellow}What's next?{nc}\n"
+        "{yellow}------------{nc}\n"
+        f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
+        f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
+        f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
+    ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
+
+    logger.info(msg)
+
+
 class CmdInit(CmdBaseNoRepo):
     def run(self):
-        from dvc.repo import Repo
         from dvc.exceptions import InitError
+        from dvc.repo import Repo
 
         try:
             self.repo = Repo.init(
@@ -19,6 +46,7 @@ class CmdInit(CmdBaseNoRepo):
                 subdir=self.args.subdir,
             )
             self.config = self.repo.config
+            _welcome_message()
         except InitError:
             logger.exception("failed to initiate DVC")
             return 1

dvc/command/install.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
@@ -10,7 +11,7 @@ class CmdInstall(CmdBase):
     def run(self):
         try:
             self.repo.install(self.args.use_pre_commit_tool)
-        except Exception:  # noqa, pylint:disable=broad-except
+        except DvcException:
             logger.exception("failed to install DVC Git hooks")
             return 1
         return 0

dvc/command/live.py

@@ -0,0 +1,94 @@
+import argparse
+import logging
+import os
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, fix_subparsers
+from dvc.utils.html import write
+
+logger = logging.getLogger(__name__)
+
+
+class CmdLive(CmdBase):
+    UNINITIALIZED = True
+
+    def _run(self, target, revs=None):
+        metrics, plots = self.repo.live.show(target=target, revs=revs)
+
+        html_path = self.args.target + ".html"
+        write(html_path, plots, metrics)
+
+        logger.info(f"\nfile://{os.path.abspath(html_path)}")
+
+        return 0
+
+
+class CmdLiveShow(CmdLive):
+    def run(self):
+        return self._run(self.args.target)
+
+
+class CmdLiveDiff(CmdLive):
+    def run(self):
+        return self._run(self.args.target, self.args.revs)
+
+
+def shared_parent_parser():
+    parent_parser = argparse.ArgumentParser(add_help=False)
+    parent_parser.add_argument(
+        "target", help="Logs dir to produce summary from",
+    ).complete = completion.DIR
+    parent_parser.add_argument(
+        "-o",
+        "--out",
+        default=None,
+        help="Destination path to save plots to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    return parent_parser
+
+
+def add_parser(subparsers, parent_parser):
+    LIVE_DESCRIPTION = (
+        "Commands to visualize and compare dvclive-produced logs."
+    )
+    live_parser = subparsers.add_parser(
+        "live",
+        parents=[parent_parser],
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=LIVE_DESCRIPTION,
+    )
+    live_subparsers = live_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc live CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(live_subparsers)
+
+    SHOW_HELP = "Visualize dvclive directory content."
+    live_show_parser = live_subparsers.add_parser(
+        "show",
+        parents=[parent_parser, shared_parent_parser()],
+        help=SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    live_show_parser.set_defaults(func=CmdLiveShow)
+
+    DIFF_HELP = (
+        "Show multiple versions of dvclive data, "
+        "by plotting it in single view."
+    )
+    live_diff_parser = live_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser, shared_parent_parser()],
+        help=DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    live_diff_parser.add_argument(
+        "--revs",
+        nargs="*",
+        default=None,
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    live_diff_parser.set_defaults(func=CmdLiveDiff)

dvc/command/metrics.py

@@ -11,34 +11,69 @@ logger = logging.getLogger(__name__)
 DEFAULT_PRECISION = 5
 
 
-def show_metrics(
-    metrics, all_branches=False, all_tags=False, all_commits=False
+def _show_metrics(
+    metrics,
+    markdown=False,
+    all_branches=False,
+    all_tags=False,
+    all_commits=False,
+    precision=None,
 ):
-    from flatten_json import flatten
-    from dvc.utils.diff import format_dict
+    from dvc.utils.diff import format_dict, table
+    from dvc.utils.flatten import flatten
 
     # When `metrics` contains a `None` key, it means that some files
     # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
     missing = metrics.pop(None, None)
+    with_rev = any([all_branches, all_tags, all_commits])
+    header_set = set()
+    rows = []
 
-    for branch, val in metrics.items():
-        if all_branches or all_tags or all_commits:
-            logger.info(f"{branch}:")
+    if precision is None:
+        precision = DEFAULT_PRECISION
 
-        for fname, metric in val.items():
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+        return val
+
+    for _branch, val in metrics.items():
+        for _fname, metric in val.items():
             if not isinstance(metric, dict):
-                logger.info("\t{}: {}".format(fname, str(metric)))
+                header_set.add("")
                 continue
+            for key, _val in flatten(format_dict(metric)).items():
+                header_set.add(key)
+    header = sorted(header_set)
+    for branch, val in metrics.items():
+        for fname, metric in val.items():
+            row = []
+            if with_rev:
+                row.append(branch)
+            row.append(fname)
+            if not isinstance(metric, dict):
+                row.append(str(metric))
+                rows.append(row)
+                continue
+            flattened_val = flatten(format_dict(metric))
 
-            logger.info(f"\t{fname}:")
-            for key, value in flatten(format_dict(metric), ".").items():
-                logger.info(f"\t\t{key}: {value}")
+            for i in header:
+                row.append(_round(flattened_val.get(i)))
+            rows.append(row)
+    header.insert(0, "Path")
+    if with_rev:
+        header.insert(0, "Revision")
 
     if missing:
         raise BadMetricError(missing)
+    return table(header, rows, markdown)
 
 
-class CmdMetricsShow(CmdBase):
+class CmdMetricsBase(CmdBase):
+    UNINITIALIZED = True
+
+
+class CmdMetricsShow(CmdMetricsBase):
     def run(self):
         try:
             metrics = self.repo.metrics.show(
@@ -54,20 +89,23 @@ class CmdMetricsShow(CmdBase):
 
                 logger.info(json.dumps(metrics))
             else:
-                show_metrics(
+                table = _show_metrics(
                     metrics,
+                    self.args.show_md,
                     self.args.all_branches,
                     self.args.all_tags,
                     self.args.all_commits,
                 )
+                if table:
+                    logger.info(table)
         except DvcException:
-            logger.exception("failed to show metrics")
+            logger.exception("")
             return 1
 
         return 0
 
 
-def _show_diff(diff, markdown=False, no_path=False, old=False, precision=None):
+def _show_diff(diff, markdown=False, no_path=False, precision=None):
     from collections import OrderedDict
 
     from dvc.utils.diff import table
@@ -87,24 +125,20 @@ def _show_diff(diff, markdown=False, no_path=False, old=False, precision=None):
         for metric, change in sorted_mdiff.items():
             row = [] if no_path else [fname]
             row.append(metric)
-            if old:
-                row.append(_round(change.get("old")))
+            row.append(_round(change.get("old")))
             row.append(_round(change["new"]))
-            row.append(_round(change.get("diff", "diff not supported")))
+            row.append(_round(change.get("diff")))
             rows.append(row)
 
     header = [] if no_path else ["Path"]
     header.append("Metric")
-    if old:
-        header.extend(["Old", "New"])
-    else:
-        header.append("Value")
+    header.extend(["Old", "New"])
     header.append("Change")
 
     return table(header, rows, markdown)
 
 
-class CmdMetricsDiff(CmdBase):
+class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
         try:
             diff = self.repo.metrics.diff(
@@ -124,7 +158,6 @@ class CmdMetricsDiff(CmdBase):
                     diff,
                     self.args.show_md,
                     self.args.no_path,
-                    self.args.old,
                     precision=self.args.precision,
                 )
                 if table:
@@ -167,8 +200,8 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help=(
-            "Limit command scope to these metric files. Using -R, "
-            "directories to search metric files in can also be given."
+            "Limit command scope to these metrics files. Using -R, "
+            "directories to search metrics files in can also be given."
         ),
     ).complete = completion.FILE
     metrics_show_parser.add_argument(
@@ -197,6 +230,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show output in JSON format.",
     )
+    metrics_show_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -204,9 +243,18 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=(
             "If any target is a directory, recursively search and process "
-            "metric files."
+            "metrics files."
         ),
     )
+    metrics_show_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics to `n` digits precision after the decimal point. "
+            f"Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
     METRICS_DIFF_HELP = (
@@ -232,8 +280,11 @@ def add_parser(subparsers, parent_parser):
         "--targets",
         nargs="*",
         help=(
-            "Limit command scope to these metric files. Using -R, "
-            "directories to search metric files in can also be given."
+            "Specific metrics file(s) to compare "
+            "(even if not found as `metrics` in `dvc.yaml`). "
+            "Using -R, directories to search metrics files in "
+            "can also be given."
+            "Shows all tracked metrics by default."
         ),
         metavar="<paths>",
     ).complete = completion.FILE
@@ -244,7 +295,7 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=(
             "If any target is a directory, recursively search and process "
-            "metric files."
+            "metrics files."
         ),
     )
     metrics_diff_parser.add_argument(
@@ -265,12 +316,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show tabulated output in the Markdown format (GFM).",
     )
-    metrics_diff_parser.add_argument(
-        "--old",
-        action="store_true",
-        default=False,
-        help="Show old metric value.",
-    )
     metrics_diff_parser.add_argument(
         "--no-path",
         action="store_true",

dvc/command/move.py

@@ -25,7 +25,7 @@ def add_parser(subparsers, parent_parser):
     MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
     MOVE_DESCRIPTION = (
         "Rename or move a DVC controlled data file or a directory.\n"
-        "It renames and modifies the corresponding DVC-file to reflect the"
+        "It renames and modifies the corresponding .dvc file to reflect the"
         " changes."
     )
 

dvc/command/params.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 from collections import OrderedDict
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 
@@ -30,11 +31,14 @@ def _show_diff(diff, markdown=False, no_path=False):
 
 
 class CmdParamsDiff(CmdBase):
+    UNINITIALIZED = True
+
     def run(self):
         try:
             diff = self.repo.params.diff(
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
+                targets=self.args.targets,
                 all=self.args.all,
             )
 
@@ -91,6 +95,16 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help=("New Git commit to compare (defaults to the current workspace)"),
     )
+    params_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help=(
+            "Specific params file(s) to compare "
+            "(even if not found as `params` in `dvc.yaml`). "
+            "Shows all tracked params by default."
+        ),
+        metavar="<paths>",
+    ).complete = completion.FILE
     params_diff_parser.add_argument(
         "--all",
         action="store_true",

dvc/command/plots.py

@@ -1,45 +1,29 @@
 import argparse
 import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.schema import PLOT_PROPS
 from dvc.utils import format_link
+from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
-PAGE_HTML = """<!DOCTYPE html>
-<html>
-<head>
-    <title>DVC Plot</title>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-</head>
-<body>
-    {divs}
-</body>
-</html>"""
-
-DIV_HTML = """<div id = "{id}"></div>
-<script type = "text/javascript">
-    var spec = {vega_json};
-    vegaEmbed('#{id}', spec);
-</script>"""
-
 
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
 
     def _props(self):
+        from dvc.schema import PLOT_PROPS
+
         # Pass only props specified by user, to not shadow ones from plot def
         props = {p: getattr(self.args, p) for p in PLOT_PROPS}
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
+        from pathlib import Path
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -58,35 +42,44 @@ class CmdPlots(CmdBase):
                 logger.info(plots[target])
                 return 0
 
-            divs = [
-                DIV_HTML.format(id=f"plot{i}", vega_json=plot)
-                for i, plot in enumerate(plots.values())
-            ]
-            html = PAGE_HTML.format(divs="\n".join(divs))
-            path = self.args.out or "plots.html"
-
-            with open(path, "w") as fobj:
-                fobj.write(html)
-
-            logger.info(
-                "file://{}".format(os.path.join(self.repo.root_dir, path))
-            )
-
+            rel: str = self.args.out or "plots.html"
+            path = (Path.cwd() / rel).resolve()
+            write(path, plots)
         except DvcException:
             logger.exception("")
             return 1
 
+        assert path.is_absolute()  # as_uri throws ValueError if not absolute
+        url = path.as_uri()
+        logger.info(url)
+        if self.args.open:
+            import webbrowser
+
+            opened = webbrowser.open(rel)
+            if not opened:
+                logger.error("Failed to open. Please try opening it manually.")
+                return 1
+
         return 0
 
 
 class CmdPlotsShow(CmdPlots):
+    UNINITIALIZED = True
+
     def _func(self, *args, **kwargs):
         return self.repo.plots.show(*args, **kwargs)
 
 
 class CmdPlotsDiff(CmdPlots):
+    UNINITIALIZED = True
+
     def _func(self, *args, **kwargs):
-        return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
+        return self.repo.plots.diff(
+            *args,
+            revs=self.args.revisions,
+            experiment=self.args.experiment,
+            **kwargs,
+        )
 
 
 class CmdPlotsModify(CmdPlots):
@@ -117,7 +110,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(plots_subparsers)
 
-    SHOW_HELP = "Generate plots from metric files."
+    SHOW_HELP = "Generate plots from metrics files."
     plots_show_parser = plots_subparsers.add_parser(
         "show",
         parents=[parent_parser],
@@ -128,7 +121,9 @@ def add_parser(subparsers, parent_parser):
     plots_show_parser.add_argument(
         "targets",
         nargs="*",
-        help="Plots files to visualize. Shows all plots by default.",
+        help="Files to visualize (supports any file, "
+        "even when not found as `plots` in `dvc.yaml`). "
+        "Shows all plots by default.",
     ).complete = completion.FILE
     _add_props_arguments(plots_show_parser)
     _add_output_arguments(plots_show_parser)
@@ -148,9 +143,20 @@ def add_parser(subparsers, parent_parser):
     plots_diff_parser.add_argument(
         "--targets",
         nargs="*",
-        help="Plots file to visualize. Shows all plots by default.",
-        metavar="<path>",
+        help=(
+            "Specific plots file(s) to visualize "
+            "(even if not found as `plots` in `dvc.yaml`). "
+            "Shows all tracked plots by default."
+        ),
+        metavar="<paths>",
     ).complete = completion.FILE
+    plots_diff_parser.add_argument(
+        "-e",
+        "--experiment",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     plots_diff_parser.add_argument(
         "revisions", nargs="*", default=None, help="Git commits to plot from",
     )
@@ -158,7 +164,7 @@ def add_parser(subparsers, parent_parser):
     _add_output_arguments(plots_diff_parser)
     plots_diff_parser.set_defaults(func=CmdPlotsDiff)
 
-    PLOTS_MODIFY_HELP = "Modify display properties of plot metric files."
+    PLOTS_MODIFY_HELP = "Modify display properties of plot metrics files."
     plots_modify_parser = plots_subparsers.add_parser(
         "modify",
         parents=[parent_parser],
@@ -231,3 +237,9 @@ def _add_output_arguments(parser):
         default=False,
         help="Show output in Vega format.",
     )
+    parser.add_argument(
+        "--open",
+        action="store_true",
+        default=False,
+        help="Open plot file directly in the browser.",
+    )

dvc/command/remote.py

@@ -3,7 +3,6 @@ import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.config import ConfigError
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -17,12 +16,16 @@ class CmdRemote(CmdConfig):
             self.args.name = self.args.name.lower()
 
     def _check_exists(self, conf):
+        from dvc.config import ConfigError
+
         if self.args.name not in conf["remote"]:
-            raise ConfigError(f"remote '{self.args.name}' doesn't exists.")
+            raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
 
 
 class CmdRemoteAdd(CmdRemote):
     def run(self):
+        from dvc.config import ConfigError
+
         if self.args.default:
             logger.info(f"Setting '{self.args.name}' as a default remote.")
 
@@ -46,13 +49,14 @@ class CmdRemoteRemove(CmdRemote):
             self._check_exists(conf)
             del conf["remote"][self.args.name]
 
+        up_to_level = self.args.level or "repo"
         # Remove core.remote refs to this remote in any shadowing configs
         for level in reversed(self.config.LEVELS):
             with self.config.edit(level) as conf:
                 if conf["core"].get("remote") == self.args.name:
                     del conf["core"]["remote"]
 
-            if level == self.args.level:
+            if level == up_to_level:
                 break
 
         return 0
@@ -60,8 +64,15 @@ class CmdRemoteRemove(CmdRemote):
 
 class CmdRemoteModify(CmdRemote):
     def run(self):
+        from dvc.config import merge
+
         with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["remote"]:
+                conf["remote"][self.args.name] = {}
             section = conf["remote"][self.args.name]
             if self.args.unset:
                 section.pop(self.args.option, None)
@@ -72,9 +83,10 @@ class CmdRemoteModify(CmdRemote):
 
 class CmdRemoteDefault(CmdRemote):
     def run(self):
+        from dvc.config import ConfigError
 
         if self.args.name is None and not self.args.unset:
-            conf = self.config.load_one(self.args.level)
+            conf = self.config.read(self.args.level)
             try:
                 print(conf["core"]["remote"])
             except KeyError:
@@ -102,7 +114,7 @@ class CmdRemoteDefault(CmdRemote):
 
 class CmdRemoteList(CmdRemote):
     def run(self):
-        conf = self.config.load_one(self.args.level)
+        conf = self.config.read(self.args.level)
         for name, conf in conf["remote"].items():
             logger.info("{}\t{}".format(name, conf["url"]))
         return 0
@@ -114,6 +126,8 @@ class CmdRemoteRename(CmdRemote):
             conf["core"]["remote"] = self.args.new
 
     def run(self):
+        from dvc.config import ConfigError
+
         all_config = self.config.load_config_to_level(None)
         if self.args.new in all_config.get("remote", {}):
             raise ConfigError(
@@ -128,8 +142,9 @@ class CmdRemoteRename(CmdRemote):
             del conf["remote"][self.args.name]
             self._rename_default(conf)
 
+        up_to_level = self.args.level or "repo"
         for level in reversed(self.config.LEVELS):
-            if level == self.args.level:
+            if level == up_to_level:
                 break
             with self.config.edit(level) as level_conf:
                 self._rename_default(level_conf)
@@ -167,7 +182,7 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at {}".format(
+        help="Remote location. See full list of supported URLs at {}".format(
             format_link("https://man.dvc.org/remote")
         ),
     )
@@ -196,7 +211,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_default_parser.add_argument(
-        "name", nargs="?", help="Name of the remote."
+        "name", nargs="?", help="Name of the remote"
     )
     remote_default_parser.add_argument(
         "-u",
@@ -215,7 +230,7 @@ def add_parser(subparsers, parent_parser):
         help=REMOTE_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remote_modify_parser.add_argument("name", help="Name of the remote.")
+    remote_modify_parser.add_argument("name", help="Name of the remote")
     remote_modify_parser.add_argument(
         "option", help="Name of the option to modify."
     )

dvc/command/remove.py

@@ -14,14 +14,15 @@ class CmdRemove(CmdBase):
             try:
                 self.repo.remove(target, outs=self.args.outs)
             except DvcException:
-                logger.exception(f"failed to remove '{target}'")
+                logger.exception("")
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     REMOVE_HELP = (
-        "Remove stage entry, remove .gitignore entry and unprotect outputs"
+        "Remove stages from dvc.yaml and/or"
+        " stop tracking files or directories."
     )
     remove_parser = subparsers.add_parser(
         "remove",
@@ -37,6 +38,8 @@ def add_parser(subparsers, parent_parser):
         help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to remove.",
+        "targets",
+        nargs="+",
+        help=".dvc files or stages from dvc.yaml to remove.",
     ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -1,79 +1,56 @@
 import argparse
 import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import show_metrics
+from dvc.command.metrics import _show_metrics
 from dvc.command.status import CmdDataStatus
-from dvc.dvcfile import PIPELINE_FILE
-from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdRepro(CmdBase):
     def run(self):
-        saved_dir = os.path.realpath(os.curdir)
-        if self.args.cwd:
-            os.chdir(self.args.cwd)
+        stages = self.repo.reproduce(**self._repro_kwargs)
+        if len(stages) == 0:
+            logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+        else:
+            logger.info(
+                "Use `dvc push` to send your updates to " "remote storage."
+            )
 
-        # Dirty hack so the for loop below can at least enter once
-        if self.args.all_pipelines:
-            self.args.targets = [None]
-        elif not self.args.targets:
-            self.args.targets = self.default_targets
+        if self.args.metrics:
+            metrics = self.repo.metrics.show()
+            logger.info(_show_metrics(metrics))
 
-        ret = 0
-        for target in self.args.targets:
-            try:
-                stages = self.repo.reproduce(
-                    target,
-                    single_item=self.args.single_item,
-                    force=self.args.force,
-                    dry=self.args.dry,
-                    interactive=self.args.interactive,
-                    pipeline=self.args.pipeline,
-                    all_pipelines=self.args.all_pipelines,
-                    run_cache=not self.args.no_run_cache,
-                    no_commit=self.args.no_commit,
-                    downstream=self.args.downstream,
-                    recursive=self.args.recursive,
-                    force_downstream=self.args.force_downstream,
-                    experiment=self.args.experiment,
-                )
+        return 0
 
-                if len(stages) == 0:
-                    logger.info(CmdDataStatus.UP_TO_DATE_MSG)
-
-                if self.args.metrics:
-                    metrics = self.repo.metrics.show()
-                    show_metrics(metrics)
-            except DvcException:
-                logger.exception("")
-                ret = 1
-                break
-
-        os.chdir(saved_dir)
-        return ret
+    @property
+    def _repro_kwargs(self):
+        return {
+            "targets": self.args.targets,
+            "single_item": self.args.single_item,
+            "force": self.args.force,
+            "dry": self.args.dry,
+            "interactive": self.args.interactive,
+            "pipeline": self.args.pipeline,
+            "all_pipelines": self.args.all_pipelines,
+            "run_cache": not self.args.no_run_cache,
+            "no_commit": self.args.no_commit,
+            "downstream": self.args.downstream,
+            "recursive": self.args.recursive,
+            "force_downstream": self.args.force_downstream,
+            "pull": self.args.pull,
+            "glob": self.args.glob,
+        }
 
 
-def add_parser(subparsers, parent_parser):
-    REPRO_HELP = (
-        "Reproduce complete or partial pipelines by executing their stages."
-    )
-    repro_parser = subparsers.add_parser(
-        "repro",
-        parents=[parent_parser],
-        description=append_doc_link(REPRO_HELP, "repro"),
-        help=REPRO_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
+def add_arguments(repro_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
-    ).complete = completion.DVC_FILE
+        help="Stages to reproduce. 'dvc.yaml' by default.",
+    ).complete = completion.DVCFILES_AND_STAGE
     repro_parser.add_argument(
         "-f",
         "--force",
@@ -89,13 +66,6 @@ def add_parser(subparsers, parent_parser):
         help="Reproduce only single data item without recursive dependencies "
         "check.",
     )
-    repro_parser.add_argument(
-        "-c",
-        "--cwd",
-        default=os.path.curdir,
-        help="Directory within your repo to reproduce from.",
-        metavar="<path>",
-    )
     repro_parser.add_argument(
         "-m",
         "--metrics",
@@ -122,8 +92,8 @@ def add_parser(subparsers, parent_parser):
         "--pipeline",
         action="store_true",
         default=False,
-        help="Reproduce the whole pipeline that the specified stage file "
-        "belongs to.",
+        help="Reproduce the whole pipeline that the specified targets "
+        "belong to.",
     )
     repro_parser.add_argument(
         "-P",
@@ -168,10 +138,32 @@ def add_parser(subparsers, parent_parser):
         help="Start from the specified stages when reproducing pipelines.",
     )
     repro_parser.add_argument(
-        "-e",
-        "--experiment",
+        "--pull",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help=(
+            "Try automatically pulling missing cache for outputs restored "
+            "from the run-cache."
+        ),
     )
+    repro_parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Allows targets containing shell-style wildcards.",
+    )
+
+
+def add_parser(subparsers, parent_parser):
+    REPRO_HELP = (
+        "Reproduce complete or partial pipelines by executing their stages."
+    )
+    repro_parser = subparsers.add_parser(
+        "repro",
+        parents=[parent_parser],
+        description=append_doc_link(REPRO_HELP, "repro"),
+        help=REPRO_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    add_arguments(repro_parser)
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/run.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.stage import parse_cmd
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -20,8 +21,9 @@ class CmdRun(CmdBase):
                 self.args.plots_no_cache,
                 self.args.outs_persist,
                 self.args.outs_persist_no_cache,
+                self.args.checkpoints,
                 self.args.params,
-                self.args.command,
+                self.args.cmd,
             ]
         ):  # pragma: no cover
             logger.error(
@@ -33,27 +35,33 @@ class CmdRun(CmdBase):
 
         try:
             self.repo.run(
-                cmd=self._parsed_cmd(),
+                cmd=parse_cmd(self.args.cmd),
                 outs=self.args.outs,
                 outs_no_cache=self.args.outs_no_cache,
                 metrics=self.args.metrics,
                 metrics_no_cache=self.args.metrics_no_cache,
                 plots=self.args.plots,
                 plots_no_cache=self.args.plots_no_cache,
+                live=self.args.live,
+                live_no_cache=self.args.live_no_cache,
+                live_no_summary=self.args.live_no_summary,
+                live_no_html=self.args.live_no_html,
                 deps=self.args.deps,
                 params=self.args.params,
                 fname=self.args.file,
                 wdir=self.args.wdir,
-                no_exec=self.args.no_exec,
+                no_exec=(self.args.no_exec or bool(self.args.checkpoints)),
                 force=self.args.force,
                 run_cache=not self.args.no_run_cache,
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
                 outs_persist_no_cache=self.args.outs_persist_no_cache,
+                checkpoints=self.args.checkpoints,
                 always_changed=self.args.always_changed,
                 name=self.args.name,
                 single_stage=self.args.single_stage,
                 external=self.args.external,
+                desc=self.args.desc,
             )
         except DvcException:
             logger.exception("")
@@ -61,30 +69,13 @@ class CmdRun(CmdBase):
 
         return 0
 
-    def _parsed_cmd(self):
-        """
-        We need to take into account two cases:
-
-        - ['python code.py foo bar']: Used mainly with dvc as a library
-        - ['echo', 'foo bar']: List of arguments received from the CLI
-
-        The second case would need quoting, as it was passed through:
-                dvc run echo "foo bar"
-        """
-        if len(self.args.command) < 2:
-            return " ".join(self.args.command)
-
-        return " ".join(self._quote_argument(arg) for arg in self.args.command)
-
-    def _quote_argument(self, argument):
-        if " " not in argument or '"' in argument:
-            return argument
-
-        return f'"{argument}"'
-
 
 def add_parser(subparsers, parent_parser):
-    RUN_HELP = "Generate a stage file from a command and execute the command."
+    from dvc.command.stage import _add_common_args
+
+    RUN_HELP = (
+        "Generate a dvc.yaml file from a command and execute the command."
+    )
     run_parser = subparsers.add_parser(
         "run",
         parents=[parent_parser],
@@ -92,93 +83,17 @@ def add_parser(subparsers, parent_parser):
         help=RUN_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    run_parser.add_argument(
-        "-d",
-        "--deps",
-        action="append",
-        default=[],
-        help="Declare dependencies for reproducible cmd.",
-        metavar="<path>",
-    )
     run_parser.add_argument(
         "-n", "--name", help="Stage name.",
     )
-    run_parser.add_argument(
-        "-o",
-        "--outs",
-        action="append",
-        default=[],
-        help="Declare output file or directory.",
-        metavar="<filename>",
-    )
-    run_parser.add_argument(
-        "-O",
-        "--outs-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory "
-        "(do not put into DVC cache).",
-        metavar="<filename>",
-    )
-    run_parser.add_argument(
-        "-p",
-        "--params",
-        action="append",
-        default=[],
-        help="Declare parameter to use as additional dependency.",
-        metavar="[<filename>:]<params_list>",
-    )
-    run_parser.add_argument(
-        "-m",
-        "--metrics",
-        action="append",
-        default=[],
-        help="Declare output metric file.",
-        metavar="<path>",
-    )
-    run_parser.add_argument(
-        "-M",
-        "--metrics-no-cache",
-        action="append",
-        default=[],
-        help="Declare output metric file (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    run_parser.add_argument(
-        "--plots",
-        action="append",
-        default=[],
-        help="Declare output plot file.",
-        metavar="<path>",
-    )
-    run_parser.add_argument(
-        "--plots-no-cache",
-        action="append",
-        default=[],
-        help="Declare output plot file (do not put into DVC cache).",
-        metavar="<path>",
-    )
     run_parser.add_argument(
         "--file", metavar="<filename>", help=argparse.SUPPRESS,
     )
-    run_parser.add_argument(
-        "-w",
-        "--wdir",
-        help="Directory within your repo to run your command in.",
-        metavar="<path>",
-    )
     run_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create stage file without actually running it.",
-    )
-    run_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite existing stage",
+        help="Only create dvc.yaml without actually running it.",
     )
     run_parser.add_argument(
         "--no-run-cache",
@@ -195,41 +110,11 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
-    run_parser.add_argument(
-        "--outs-persist",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro.",
-        metavar="<filename>",
-    )
-    run_parser.add_argument(
-        "--outs-persist-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro (do not put into DVC cache).",
-        metavar="<filename>",
-    )
-    run_parser.add_argument(
-        "--always-changed",
-        action="store_true",
-        default=False,
-        help="Always consider this DVC-file as changed.",
-    )
     run_parser.add_argument(
         "--single-stage",
         action="store_true",
         default=False,
         help=argparse.SUPPRESS,
     )
-    run_parser.add_argument(
-        "--external",
-        action="store_true",
-        default=False,
-        help="Allow outputs that are outside of the DVC repository.",
-    )
-    run_parser.add_argument(
-        "command", nargs=argparse.REMAINDER, help="Command to execute."
-    )
+    _add_common_args(run_parser)
     run_parser.set_defaults(func=CmdRun)

dvc/command/stage.py

@@ -0,0 +1,365 @@
+import argparse
+import logging
+from itertools import chain, filterfalse
+from typing import TYPE_CHECKING, Dict, Iterable, List
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.utils.cli_parse import parse_params
+from dvc.utils.humanize import truncate_text
+
+if TYPE_CHECKING:
+    from dvc.output.base import BaseOutput
+    from dvc.stage import Stage
+
+logger = logging.getLogger(__name__)
+
+MAX_TEXT_LENGTH = 80
+
+
+def generate_description(stage: "Stage") -> str:
+    def part_desc(outs: Iterable["BaseOutput"]) -> str:
+        return ", ".join(out.def_path for out in outs)
+
+    if not stage.deps and not stage.outs:
+        return "No outputs or dependencies"
+
+    if not stage.outs and stage.deps:
+        return "Depends on " + part_desc(stage.deps)
+
+    def is_plot_or_metric(out: "BaseOutput"):
+        return bool(out.plot) or bool(out.metric)
+
+    desc: List[str] = []
+
+    outs = list(filterfalse(is_plot_or_metric, stage.outs))
+    if outs:
+        desc.append("Outputs " + part_desc(outs))
+
+    plots_and_metrics = list(filter(is_plot_or_metric, stage.outs))
+    if plots_and_metrics:
+        desc.append("Reports " + part_desc(plots_and_metrics))
+
+    return "; ".join(desc)
+
+
+def prepare_description(
+    stage: "Stage", max_length: int = MAX_TEXT_LENGTH
+) -> str:
+    desc = stage.short_description() or generate_description(stage)
+    return truncate_text(desc, max_length)
+
+
+def prepare_stages_data(
+    stages: Iterable["Stage"],
+    description: bool = True,
+    max_length: int = MAX_TEXT_LENGTH,
+) -> Dict[str, str]:
+    return {
+        stage.addressing: prepare_description(stage, max_length=max_length)
+        if description
+        else ""
+        for stage in stages
+    }
+
+
+class CmdStageList(CmdBase):
+    def _get_stages(self) -> Iterable["Stage"]:
+        if self.args.all:
+            stages: List["Stage"] = self.repo.stages  # type: ignore
+            logger.trace(  # type: ignore[attr-defined]
+                "%d no. of stages found", len(stages)
+            )
+            return stages
+
+        # removing duplicates while maintaining order
+        collected = chain.from_iterable(
+            self.repo.stage.collect(
+                target=target,
+                recursive=self.args.recursive,
+                accept_group=True,
+            )
+            for target in self.args.targets
+        )
+        return dict.fromkeys(collected).keys()
+
+    def run(self):
+        from dvc.utils.diff import table
+
+        def log_error(relpath: str, exc: Exception):
+            if self.args.fail:
+                raise exc
+            logger.debug("Stages from %s failed to load", relpath)
+
+        # silence stage collection error by default
+        self.repo.stage_collection_error_handler = log_error
+
+        stages = self._get_stages()
+        names_only = self.args.names_only
+
+        data = prepare_stages_data(stages, description=not names_only)
+        if data:
+            print(table(header=(), rows=data.items()))
+
+        return 0
+
+
+def parse_cmd(commands: List[str]) -> str:
+    """
+    We need to take into account two cases:
+
+    - ['python code.py foo bar']: Used mainly with dvc as a library
+    - ['echo', 'foo bar']: List of arguments received from the CLI
+
+    The second case would need quoting, as it was passed through:
+            dvc run echo "foo bar"
+    """
+
+    def quote_argument(arg: str):
+        should_quote = " " in arg and '"' not in arg
+        return f'"{arg}"' if should_quote else arg
+
+    if len(commands) < 2:
+        return " ".join(commands)
+    return " ".join(map(quote_argument, commands))
+
+
+class CmdStageAdd(CmdBase):
+    def run(self):
+        kwargs = vars(self.args)
+        kwargs.update(
+            {
+                "cmd": parse_cmd(kwargs.pop("cmd")),
+                "params": parse_params(self.args.params),
+            }
+        )
+        self.repo.stage.add(**kwargs)
+        return 0
+
+
+def _add_common_args(parser):
+    parser.add_argument(
+        "-d",
+        "--deps",
+        action="append",
+        default=[],
+        help="Declare dependencies for reproducible cmd.",
+        metavar="<path>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-o",
+        "--outs",
+        action="append",
+        default=[],
+        help="Declare output file or directory.",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-O",
+        "--outs-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory "
+        "(do not put into DVC cache).",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-p",
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter to use as additional dependency.",
+        metavar="[<filename>:]<params_list>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-m",
+        "--metrics",
+        action="append",
+        default=[],
+        help="Declare output metrics file.",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "-M",
+        "--metrics-no-cache",
+        action="append",
+        default=[],
+        help="Declare output metrics file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--plots",
+        action="append",
+        default=[],
+        help="Declare output plot file.",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--plots-no-cache",
+        action="append",
+        default=[],
+        help="Declare output plot file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--live", help="Declare output as dvclive.", metavar="<path>",
+    )
+    parser.add_argument(
+        "--live-no-cache",
+        help="Declare output as dvclive (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--live-no-summary",
+        action="store_true",
+        default=False,
+        help="Signal dvclive logger to not dump latest metrics file.",
+    )
+    parser.add_argument(
+        "--live-no-html",
+        action="store_true",
+        default=False,
+        help="Signal dvclive logger to not produce training report.",
+    )
+    parser.add_argument(
+        "-w",
+        "--wdir",
+        help="Directory within your repo to run your command in.",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite existing stage",
+    )
+    parser.add_argument(
+        "--outs-persist",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro.",
+        metavar="<filename>",
+    )
+    parser.add_argument(
+        "--outs-persist-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro (do not put into DVC cache).",
+        metavar="<filename>",
+    )
+    parser.add_argument(
+        "-c",
+        "--checkpoints",
+        action="append",
+        default=[],
+        help="Declare checkpoint output file or directory for 'dvc exp run'. "
+        "Not compatible with 'dvc repro'.",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "--always-changed",
+        action="store_true",
+        default=False,
+        help="Always consider this DVC-file as changed.",
+    )
+    parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow outputs that are outside of the DVC repository.",
+    )
+    parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the stage (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    parser.add_argument(
+        "cmd",
+        nargs=argparse.REMAINDER,
+        help="Command to execute.",
+        metavar="command",
+    )
+
+
+def add_parser(subparsers, parent_parser):
+    STAGES_HELP = "Commands to list and create stages."
+
+    stage_parser = subparsers.add_parser(
+        "stage",
+        parents=[parent_parser],
+        description=append_doc_link(STAGES_HELP, "stage"),
+        help=STAGES_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    stage_subparsers = stage_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc stage CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(stage_subparsers)
+
+    STAGE_ADD_HELP = "Create stage"
+    stage_add_parser = stage_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(STAGE_ADD_HELP, "stage/add"),
+        help=STAGE_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    stage_add_parser.add_argument(
+        "-n", "--name", help="Name of the stage to add", required=True
+    )
+    _add_common_args(stage_add_parser)
+    stage_add_parser.set_defaults(func=CmdStageAdd)
+
+    STAGE_LIST_HELP = "List stages."
+    stage_list_parser = stage_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(STAGE_LIST_HELP, "stage/list"),
+        help=STAGE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    stage_list_parser.add_argument(
+        "targets",
+        nargs="*",
+        default=["dvc.yaml"],
+        help=(
+            "Show stages from a dvc.yaml/.dvc file or a directory. "
+            "'dvc.yaml' by default"
+        ),
+    )
+    stage_list_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="List all of the stages in the repo.",
+    )
+    stage_list_parser.add_argument(
+        "--fail",
+        action="store_true",
+        default=False,
+        help="Fail immediately, do not suppress any syntax errors.",
+    )
+    stage_list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="List all stages inside the specified directory.",
+    )
+    stage_list_parser.add_argument(
+        "--names-only",
+        action="store_true",
+        default=False,
+        help="List only stage names.",
+    )
+    stage_list_parser.set_defaults(func=CmdStageList)

dvc/command/status.py

@@ -1,6 +1,8 @@
 import logging
 
 from dvc.command.data_sync import CmdDataBase
+from dvc.exceptions import DvcException
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
@@ -9,6 +11,11 @@ class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 20
     STATUS_INDENT = "\t"
     UP_TO_DATE_MSG = "Data and pipelines are up to date."
+    IN_SYNC_MSG = "Cache and remote '{remote}' are in sync."
+    EMPTY_PROJECT_MSG = (
+        "There are no data or pipelines tracked in this project yet.\n"
+        "See {link} to get started!"
+    ).format(link=format_link("https://dvc.org/doc/start"))
 
     def _normalize(self, s):
         s += ":"
@@ -37,6 +44,8 @@ class CmdDataStatus(CmdDataBase):
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.ui import ui
+
         indent = 1 if self.args.cloud else 0
         try:
             st = self.repo.status(
@@ -60,10 +69,17 @@ class CmdDataStatus(CmdDataBase):
                 logger.info(json.dumps(st))
             elif st:
                 self._show(st, indent)
+            elif not self.repo.stages:
+                ui.write(self.EMPTY_PROJECT_MSG)
+            elif self.args.cloud or self.args.remote:
+                remote = self.args.remote or self.repo.config["core"].get(
+                    "remote"
+                )
+                ui.write(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                logger.info(self.UP_TO_DATE_MSG)
+                ui.write(self.UP_TO_DATE_MSG)
 
-        except Exception:  # noqa, pylint: disable=broad-except
-            logger.exception("failed to obtain data status")
+        except DvcException:
+            logger.exception("")
             return 1
         return 0

dvc/command/update.py

@@ -16,6 +16,9 @@ class CmdUpdate(CmdBase):
                 targets=self.args.targets,
                 rev=self.args.rev,
                 recursive=self.args.recursive,
+                to_remote=self.args.to_remote,
+                remote=self.args.remote,
+                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception("failed update data")
@@ -33,7 +36,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to update.",
+        "targets", nargs="+", help=".dvc files to update.",
     ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",
@@ -48,4 +51,27 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Update all stages in the specified directory.",
     )
+    update_parser.add_argument(
+        "--to-remote",
+        action="store_true",
+        default=False,
+        help="Update data directly on the remote",
+    )
+    update_parser.add_argument(
+        "-r",
+        "--remote",
+        help="Remote storage to perform updates to",
+        metavar="<name>",
+    )
+    update_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -1,146 +1,19 @@
 import argparse
-import itertools
 import logging
-import os
-import pathlib
-import platform
-import uuid
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.scm.base import SCMError
-from dvc.system import System
-from dvc.utils import is_binary, relpath
-from dvc.utils.pkg import PKG
-from dvc.version import __version__
-
-try:
-    import psutil
-except ImportError:
-    psutil = None
-
 
 logger = logging.getLogger(__name__)
 
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
-        from dvc.repo import Repo
+        from dvc.info import get_dvc_info
 
-        info = [
-            f"DVC version: {__version__}",
-            f"Python version: {platform.python_version()}",
-            f"Platform: {platform.platform()}",
-            f"Binary: {is_binary()}",
-            f"Package: {PKG}",
-            f"Supported remotes: {self.get_supported_remotes()}",
-        ]
-
-        try:
-            repo = Repo()
-            root_directory = repo.root_dir
-
-            # cache_dir might not exist yet (e.g. after `dvc init`), and we
-            # can't auto-create it, as it might cause issues if the user
-            # later decides to enable shared cache mode with
-            # `dvc config cache.shared group`.
-            if os.path.exists(repo.cache.local.cache_dir):
-                info.append(
-                    "Cache: {}".format(self.get_linktype_support_info(repo))
-                )
-                if psutil:
-                    fs_type = self.get_fs_type(repo.cache.local.cache_dir)
-                    info.append(
-                        f"Filesystem type (cache directory): {fs_type}"
-                    )
-            else:
-                logger.warning(
-                    "Unable to detect supported link types, as cache "
-                    "directory '{}' doesn't exist. It is usually auto-created "
-                    "by commands such as `dvc add/fetch/pull/run/import`, "
-                    "but you could create it manually to enable this "
-                    "check.".format(relpath(repo.cache.local.cache_dir))
-                )
-
-        except NotDvcRepoError:
-            root_directory = os.getcwd()
-        except SCMError:
-            root_directory = os.getcwd()
-            info.append("Repo: dvc, git (broken)")
-        else:
-            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
-
-        if psutil:
-            fs_root = self.get_fs_type(os.path.abspath(root_directory))
-            info.append(f"Filesystem type (workspace): {fs_root}")
-
-        logger.info("\n".join(info))
+        dvc_info = get_dvc_info()
+        logger.info(dvc_info)
         return 0
 
-    @staticmethod
-    def get_fs_type(path):
-        partition = {
-            pathlib.Path(part.mountpoint): (part.fstype, part.device)
-            for part in psutil.disk_partitions(all=True)
-        }
-
-        path = pathlib.Path(path)
-
-        for parent in itertools.chain([path], path.parents):
-            if parent in partition:
-                return partition[parent]
-        return ("unknown", "none")
-
-    @staticmethod
-    def get_linktype_support_info(repo):
-        links = {
-            "reflink": (System.reflink, None),
-            "hardlink": (System.hardlink, System.is_hardlink),
-            "symlink": (System.symlink, System.is_symlink),
-        }
-
-        fname = "." + str(uuid.uuid4())
-        src = os.path.join(repo.cache.local.cache_dir, fname)
-        open(src, "w").close()
-        dst = os.path.join(repo.root_dir, fname)
-
-        cache = []
-
-        for name, (link, is_link) in links.items():
-            try:
-                link(src, dst)
-                status = "supported"
-                if is_link and not is_link(dst):
-                    status = "broken"
-                os.unlink(dst)
-            except DvcException:
-                status = "not supported"
-            cache.append(f"{name} - {status}")
-        os.remove(src)
-
-        return ", ".join(cache)
-
-    @staticmethod
-    def get_supported_remotes():
-        from dvc.tree import TREES
-
-        supported_remotes = []
-        for tree_cls in TREES:
-            if not tree_cls.get_missing_deps():
-                supported_remotes.append(tree_cls.scheme)
-
-        return ", ".join(supported_remotes)
-
-
-def _get_dvc_repo_info(repo):
-    if repo.config.get("core", {}).get("no_scm", False):
-        return "dvc (no_scm)"
-
-    if repo.root_dir != repo.scm.root_dir:
-        return "dvc (subdir), git"
-
-    return "dvc, git"
-
 
 def add_parser(subparsers, parent_parser):
     VERSION_HELP = (
@@ -152,5 +25,6 @@ def add_parser(subparsers, parent_parser):
         description=append_doc_link(VERSION_HELP, "version"),
         help=VERSION_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
+        aliases=["doctor"],
     )
     version_parser.set_defaults(func=CmdVersion)

dvc/config.py

@@ -4,25 +4,11 @@ import os
 import re
 from contextlib import contextmanager
 from functools import partial
-from urllib.parse import urlparse
 
-import configobj
-from funcy import cached_property, compact, re_find, walk_values
-from voluptuous import (
-    ALLOW_EXTRA,
-    All,
-    Any,
-    Coerce,
-    Invalid,
-    Lower,
-    Optional,
-    Range,
-    Schema,
-)
+from funcy import cached_property, compact, memoize, re_find
 
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.path_info import PathInfo
-from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -38,177 +24,19 @@ class NoRemoteError(ConfigError):
     pass
 
 
-def supported_cache_type(types):
-    """Checks if link type config option consists only of valid values.
+@memoize
+def get_compiled_schema():
+    from voluptuous import Schema
 
-    Args:
-        types (list/string): type(s) of links that dvc should try out.
-    """
-    if types is None:
-        return None
-    if isinstance(types, str):
-        types = [typ.strip() for typ in types.split(",")]
+    from .config_schema import SCHEMA
 
-    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
-    if unsupported:
-        raise Invalid(
-            "Unsupported cache type(s): {}".format(", ".join(unsupported))
-        )
-
-    return types
+    return Schema(SCHEMA)
 
 
-# Checks that value is either true or false and converts it to bool
-to_bool = Bool = All(
-    Lower,
-    Any("true", "false"),
-    lambda v: v == "true",
-    msg="expected true or false",
-)
+def to_bool(value):
+    from .config_schema import Bool
 
-
-def Choices(*choices):
-    """Checks that value belongs to the specified set of values
-
-    Args:
-        *choices: pass allowed values as arguments, or pass a list or
-            tuple as a single argument
-    """
-    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
-
-
-def ByUrl(mapping):
-    schemas = walk_values(Schema, mapping)
-
-    def validate(data):
-        if "url" not in data:
-            raise Invalid("expected 'url'")
-
-        parsed = urlparse(data["url"])
-        # Windows absolute paths should really have scheme == "" (local)
-        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
-            return schemas[""](data)
-        if parsed.scheme not in schemas:
-            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
-
-        return schemas[parsed.scheme](data)
-
-    return validate
-
-
-class RelPath(str):
-    pass
-
-
-REMOTE_COMMON = {
-    "url": str,
-    "checksum_jobs": All(Coerce(int), Range(1)),
-    Optional("no_traverse"): Bool,  # obsoleted
-    "verify": Bool,
-}
-LOCAL_COMMON = {
-    "type": supported_cache_type,
-    Optional("protected", default=False): Bool,  # obsoleted
-    "shared": All(Lower, Choices("group")),
-    Optional("slow_link_warning", default=True): Bool,
-}
-HTTP_COMMON = {
-    "auth": All(Lower, Choices("basic", "digest", "custom")),
-    "custom_auth_header": str,
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-}
-SCHEMA = {
-    "core": {
-        "remote": Lower,
-        "checksum_jobs": All(Coerce(int), Range(1)),
-        Optional("interactive", default=False): Bool,
-        Optional("analytics", default=True): Bool,
-        Optional("hardlink_lock", default=False): Bool,
-        Optional("no_scm", default=False): Bool,
-        Optional("experiments", default=False): Bool,
-    },
-    "cache": {
-        "local": str,
-        "s3": str,
-        "gs": str,
-        "hdfs": str,
-        "ssh": str,
-        "azure": str,
-        # This is for default local cache
-        "dir": str,
-        **LOCAL_COMMON,
-    },
-    "remote": {
-        str: ByUrl(
-            {
-                "": {**LOCAL_COMMON, **REMOTE_COMMON},
-                "s3": {
-                    "region": str,
-                    "profile": str,
-                    "credentialpath": str,
-                    "endpointurl": str,
-                    "access_key_id": str,
-                    "secret_access_key": str,
-                    Optional("listobjects", default=False): Bool,
-                    Optional("use_ssl", default=True): Bool,
-                    "sse": str,
-                    "sse_kms_key_id": str,
-                    "acl": str,
-                    "grant_read": str,
-                    "grant_read_acp": str,
-                    "grant_write_acp": str,
-                    "grant_full_control": str,
-                    **REMOTE_COMMON,
-                },
-                "gs": {
-                    "projectname": str,
-                    "credentialpath": str,
-                    **REMOTE_COMMON,
-                },
-                "ssh": {
-                    "type": supported_cache_type,
-                    "port": Coerce(int),
-                    "user": str,
-                    "password": str,
-                    "ask_password": Bool,
-                    "keyfile": str,
-                    "timeout": Coerce(int),
-                    "gss_auth": Bool,
-                    **REMOTE_COMMON,
-                },
-                "hdfs": {"user": str, **REMOTE_COMMON},
-                "azure": {"connection_string": str, **REMOTE_COMMON},
-                "oss": {
-                    "oss_key_id": str,
-                    "oss_key_secret": str,
-                    "oss_endpoint": str,
-                    **REMOTE_COMMON,
-                },
-                "gdrive": {
-                    "gdrive_use_service_account": Bool,
-                    "gdrive_client_id": str,
-                    "gdrive_client_secret": str,
-                    "gdrive_user_credentials_file": str,
-                    "gdrive_service_account_email": str,
-                    "gdrive_service_account_user_email": str,
-                    "gdrive_service_account_p12_file_path": str,
-                    Optional("gdrive_trash_only", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "http": {**HTTP_COMMON, **REMOTE_COMMON},
-                "https": {**HTTP_COMMON, **REMOTE_COMMON},
-                "remote": {str: object},  # Any of the above options are valid
-            }
-        )
-    },
-    "state": {
-        "row_limit": All(Coerce(int), Range(1)),
-        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
-    },
-}
-COMPILED_SCHEMA = Schema(SCHEMA)
+    return Bool(value)
 
 
 class Config(dict):
@@ -235,9 +63,9 @@ class Config(dict):
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True, tree=None,
+        self, dvc_dir=None, validate=True, fs=None, config=None,
     ):  # pylint: disable=super-init-not-called
-        from dvc.tree.local import LocalTree
+        from dvc.fs.local import LocalFileSystem
 
         self.dvc_dir = dvc_dir
 
@@ -251,14 +79,14 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.wtree = LocalTree(None, {"url": self.dvc_dir})
-        self.tree = tree or self.wtree
+        self.wfs = LocalFileSystem(None, {"url": self.dvc_dir})
+        self.fs = fs or self.wfs
 
-        self.load(validate=validate)
+        self.load(validate=validate, config=config)
 
     @classmethod
     def get_dir(cls, level):
-        from appdirs import user_config_dir, site_config_dir
+        from appdirs import site_config_dir, user_config_dir
 
         assert level in ("global", "system")
 
@@ -294,7 +122,7 @@ class Config(dict):
         open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def load(self, validate=True):
+    def load(self, validate=True, config=None):
         """Loads config from all the config files.
 
         Raises:
@@ -302,6 +130,9 @@ class Config(dict):
         """
         conf = self.load_config_to_level()
 
+        if config is not None:
+            merge(conf, config)
+
         if validate:
             conf = self.validate(conf)
 
@@ -312,25 +143,36 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
-    def _load_config(self, level):
-        filename = self.files[level]
+    def _get_fs(self, level):
+        # NOTE: this might be a Gitfs, which doesn't see things outside of
+        # the repo.
+        return self.fs if level == "repo" else self.wfs
 
-        if self.tree.exists(filename, use_dvcignore=False):
-            with self.tree.open(filename) as fobj:
-                conf_obj = configobj.ConfigObj(fobj)
+    def _load_config(self, level):
+        from configobj import ConfigObj
+
+        filename = self.files[level]
+        fs = self._get_fs(level)
+
+        if fs.exists(filename, use_dvcignore=False):
+            with fs.open(filename) as fobj:
+                conf_obj = ConfigObj(fobj)
         else:
-            conf_obj = configobj.ConfigObj()
+            conf_obj = ConfigObj()
         return _parse_remotes(_lower_keys(conf_obj.dict()))
 
     def _save_config(self, level, conf_dict):
+        from configobj import ConfigObj
+
         filename = self.files[level]
+        fs = self._get_fs(level)
 
         logger.debug(f"Writing '{filename}'.")
 
-        self.tree.makedirs(os.path.dirname(filename))
+        fs.makedirs(os.path.dirname(filename))
 
-        config = configobj.ConfigObj(_pack_remotes(conf_dict))
-        with self.tree.open(filename, "wb") as fobj:
+        config = ConfigObj(_pack_remotes(conf_dict))
+        with fs.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
 
@@ -339,7 +181,7 @@ class Config(dict):
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
-        for key in COMPILED_SCHEMA.schema:
+        for key in get_compiled_schema().schema:
             conf.setdefault(key, {})
 
         return conf
@@ -349,14 +191,26 @@ class Config(dict):
         abs_conf_dir = os.path.abspath(os.path.dirname(filename))
 
         def resolve(path):
+            from .config_schema import RelPath
+
             if os.path.isabs(path) or re.match(r"\w+://", path):
                 return path
+
+            # on windows convert slashes to backslashes
+            # to have path compatible with abs_conf_dir
+            if os.path.sep == "\\" and "/" in path:
+                path = path.replace("/", "\\")
+
             return RelPath(os.path.join(abs_conf_dir, path))
 
         return Config._map_dirs(conf, resolve)
 
     @staticmethod
     def _to_relpath(conf_dir, path):
+        from dvc.utils import relpath
+
+        from .config_schema import RelPath
+
         if re.match(r"\w+://", path):
             return path
 
@@ -374,10 +228,20 @@ class Config(dict):
 
     @staticmethod
     def _map_dirs(conf, func):
+        from voluptuous import ALLOW_EXTRA, Schema
+
         dirs_schema = {
             "cache": {"dir": func},
             "remote": {
-                str: {"url": func, "gdrive_user_credentials_file": func}
+                str: {
+                    "url": func,
+                    "gdrive_user_credentials_file": func,
+                    "gdrive_service_account_json_file_path": func,
+                    "credentialpath": func,
+                    "keyfile": func,
+                    "cert_path": func,
+                    "key_path": func,
+                }
             },
         }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
@@ -388,11 +252,19 @@ class Config(dict):
             if merge_level == level:
                 break
             if merge_level in self.files:
-                _merge(merged_conf, self.load_one(merge_level))
+                merge(merged_conf, self.load_one(merge_level))
         return merged_conf
 
+    def read(self, level=None):
+        # NOTE: we read from a merged config by default, same as git config
+        if level is None:
+            return self.load_config_to_level()
+        return self.load_one(level)
+
     @contextmanager
-    def edit(self, level="repo"):
+    def edit(self, level=None, validate=True):
+        # NOTE: we write to repo config by default, same as git config
+        level = level or "repo"
         if level in {"repo", "local"} and self.dvc_dir is None:
             raise ConfigError("Not inside a DVC repo")
 
@@ -402,16 +274,20 @@ class Config(dict):
         conf = self._save_paths(conf, self.files[level])
 
         merged_conf = self.load_config_to_level(level)
-        _merge(merged_conf, conf)
-        self.validate(merged_conf)
+        merge(merged_conf, conf)
+
+        if validate:
+            self.validate(merged_conf)
 
         self._save_config(level, conf)
-        self.load()
+        self.load(validate=validate)
 
     @staticmethod
     def validate(data):
+        from voluptuous import Invalid
+
         try:
-            return COMPILED_SCHEMA(data)
+            return get_compiled_schema()(data)
         except Invalid as exc:
             raise ConfigError(str(exc)) from None
 
@@ -441,11 +317,11 @@ def _pack_remotes(conf):
     return result
 
 
-def _merge(into, update):
+def merge(into, update):
     """Merges second dict into first recursively"""
     for key, val in update.items():
         if isinstance(into.get(key), dict) and isinstance(val, dict):
-            _merge(into[key], val)
+            merge(into[key], val)
         else:
             into[key] = val
 

dvc/config_schema.py

@@ -0,0 +1,225 @@
+import os
+from urllib.parse import urlparse
+
+from funcy import walk_values
+from voluptuous import (
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
+
+Bool = All(
+    Lower,
+    Any("true", "false"),
+    lambda v: v == "true",
+    msg="expected true or false",
+)
+
+
+def supported_cache_type(types):
+    """Checks if link type config option consists only of valid values.
+
+    Args:
+        types (list/string): type(s) of links that dvc should try out.
+    """
+    if types is None:
+        return None
+    if isinstance(types, str):
+        types = [typ.strip() for typ in types.split(",")]
+
+    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
+    if unsupported:
+        raise Invalid(
+            "Unsupported cache type(s): {}".format(", ".join(unsupported))
+        )
+
+    return types
+
+
+def Choices(*choices):
+    """Checks that value belongs to the specified set of values
+
+    Args:
+        *choices: pass allowed values as arguments, or pass a list or
+            tuple as a single argument
+    """
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
+
+
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    "jobs": All(Coerce(int), Range(1)),
+    Optional("no_traverse"): Bool,  # obsoleted
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "ssl_verify": Bool,
+    "method": str,
+}
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+}
+
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+        Optional("autostage", default=False): Bool,
+        Optional("experiments"): Bool,  # obsoleted
+        Optional("check_update", default=True): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "webhdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "configpath": str,
+                    "endpointurl": str,
+                    "access_key_id": str,
+                    "secret_access_key": str,
+                    "session_token": str,
+                    Optional("listobjects", default=False): Bool,  # obsoleted
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "sse_kms_key_id": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    "allow_agent": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "webhdfs": {
+                    "hdfscli_config": str,
+                    "webhdfs_token": str,
+                    "user": str,
+                    "webhdfs_alias": str,
+                    **REMOTE_COMMON,
+                },
+                "azure": {
+                    "connection_string": str,
+                    "sas_token": str,
+                    "account_name": str,
+                    "account_key": str,
+                    "tenant_id": str,
+                    "client_id": str,
+                    "client_secret": str,
+                    **REMOTE_COMMON,
+                },
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_json_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),  # obsoleted
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
+    },
+    # section for experimental features
+    "feature": {
+        # enabled by default. It's of no use, kept for backward compatibility.
+        Optional("parametrization", default=True): Bool
+    },
+}

dvc/daemon.py

@@ -25,7 +25,7 @@ def _popen(cmd, **kwargs):
 
 
 def _spawn_windows(cmd, env):
-    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
 
     creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 

dvc/data_cloud.py

@@ -2,9 +2,6 @@
 
 import logging
 
-from dvc.config import NoRemoteError
-from dvc.remote import get_remote
-
 logger = logging.getLogger(__name__)
 
 
@@ -23,6 +20,8 @@ class DataCloud:
         self.repo = repo
 
     def get_remote(self, name=None, command="<command>"):
+        from dvc.config import NoRemoteError
+
         if not name:
             name = self.repo.config["core"].get("remote")
 
@@ -45,6 +44,8 @@ class DataCloud:
         raise NoRemoteError(error_msg)
 
     def _init_remote(self, name):
+        from dvc.remote import get_remote
+
         return get_remote(self.repo, name=name)
 
     def push(
@@ -62,8 +63,11 @@ class DataCloud:
         """
         remote = self.get_remote(remote, "push")
 
-        return self.repo.cache.local.push(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
+        return remote.push(
+            self.repo.odb.local,
+            cache,
+            jobs=jobs,
+            show_checksums=show_checksums,
         )
 
     def pull(
@@ -81,26 +85,39 @@ class DataCloud:
         """
         remote = self.get_remote(remote, "pull")
 
-        downloaded_items_num = self.repo.cache.local.pull(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
+        return remote.pull(
+            self.repo.odb.local,
+            cache,
+            jobs=jobs,
+            show_checksums=show_checksums,
         )
 
-        if not remote.tree.verify:
-            self._save_pulled_checksums(cache)
+    def transfer(self, source, jobs=None, remote=None, command=None):
+        """Transfer data items in a cloud-agnostic way.
 
-        return downloaded_items_num
+        Args:
+            source (str): url for the source location.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.BaseRemote): optional remote to compare
+                cache to. By default remote from core.remote config option
+                is used.
+            command (str): the command which is benefitting from this function
+                (to be used for reporting better error messages).
+        """
+        from dvc.fs import get_cloud_fs
 
-    def _save_pulled_checksums(self, cache):
-        for checksum in cache.scheme_keys("local"):
-            cache_file = self.repo.cache.local.tree.hash_to_path_info(checksum)
-            if self.repo.cache.local.tree.exists(cache_file):
-                # We can safely save here, as existing corrupted files will
-                # be removed upon status, while files corrupted during
-                # download will not be moved from tmp_file
-                # (see `RemoteBASE.download()`)
-                self.repo.state.save(cache_file, checksum)
+        from_fs = get_cloud_fs(self.repo, url=source)
+        remote = self.get_remote(remote, command)
+        return remote.transfer(from_fs, from_fs.path_info, jobs=jobs)
 
-    def status(self, cache, jobs=None, remote=None, show_checksums=False):
+    def status(
+        self,
+        cache,
+        jobs=None,
+        remote=None,
+        show_checksums=False,
+        log_missing=True,
+    ):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
@@ -111,8 +128,18 @@ class DataCloud:
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
+            log_missing (bool): log warning messages if file doesn't exist
+                neither in cache, neither in cloud.
         """
         remote = self.get_remote(remote, "status")
-        return self.repo.cache.local.status(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
+        return remote.status(
+            self.repo.odb.local,
+            cache,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            log_missing=log_missing,
         )
+
+    def get_url_for(self, remote, checksum):
+        remote = self.get_remote(remote)
+        return str(remote.odb.hash_to_path_info(checksum))

dvc/dependency/__init__.py

@@ -11,23 +11,15 @@ from dvc.dependency.local import LocalDependency
 from dvc.dependency.param import ParamsDependency
 from dvc.dependency.s3 import S3Dependency
 from dvc.dependency.ssh import SSHDependency
+from dvc.dependency.webdav import WebDAVDependency
+from dvc.dependency.webdavs import WebDAVSDependency
+from dvc.dependency.webhdfs import WebHDFSDependency
 from dvc.output.base import BaseOutput
 from dvc.scheme import Schemes
 
-from ..tree import get_cloud_tree
+from ..fs import get_cloud_fs
 from .repo import RepoDependency
 
-DEPS = [
-    AzureDependency,
-    GSDependency,
-    HDFSDependency,
-    HTTPDependency,
-    HTTPSDependency,
-    S3Dependency,
-    SSHDependency,
-    # NOTE: LocalDependency is the default choice
-]
-
 DEP_MAP = {
     Schemes.LOCAL: LocalDependency,
     Schemes.SSH: SSHDependency,
@@ -37,16 +29,20 @@ DEP_MAP = {
     Schemes.HDFS: HDFSDependency,
     Schemes.HTTP: HTTPDependency,
     Schemes.HTTPS: HTTPSDependency,
+    Schemes.WEBDAV: WebDAVDependency,
+    Schemes.WEBDAVS: WebDAVSDependency,
+    Schemes.WEBHDFS: WebHDFSDependency,
 }
 
 
 # NOTE: schema for dependencies is basically the same as for outputs, but
 # without output-specific entries like 'cache' (whether or not output is
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
-# output is a metric file and how to parse it, see `-M` flag for `dvc run`).
+# output is a metrics file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
 del SCHEMA[BaseOutput.PARAM_CACHE]
 del SCHEMA[BaseOutput.PARAM_METRIC]
+del SCHEMA[BaseOutput.PARAM_DESC]
 SCHEMA.update(RepoDependency.REPO_SCHEMA)
 SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 
@@ -54,8 +50,8 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 def _get(stage, p, info):
     parsed = urlparse(p) if p else None
     if parsed and parsed.scheme == "remote":
-        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
-        return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
+        fs = get_cloud_fs(stage.repo, name=parsed.netloc)
+        return DEP_MAP[fs.scheme](stage, p, info, fs=fs)
 
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)
@@ -65,10 +61,8 @@ def _get(stage, p, info):
         params = info.pop(ParamsDependency.PARAM_PARAMS)
         return ParamsDependency(stage, p, params)
 
-    for d in DEPS:
-        if d.supported(p):
-            return d(stage, p, info)
-    return LocalDependency(stage, p, info)
+    dep_cls = DEP_MAP.get(parsed.scheme, LocalDependency)
+    return dep_cls(stage, p, info)
 
 
 def loadd_from(stage, d_list):

dvc/dependency/azure.py

@@ -1,8 +1,8 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
 
-from ..tree.azure import AzureTree
+from ..fs.azure import AzureFileSystem
 
 
 class AzureDependency(BaseDependency, BaseOutput):
-    TREE_CLS = AzureTree
+    FS_CLS = AzureFileSystem

dvc/dependency/base.py

@@ -1,3 +1,5 @@
+from typing import Type
+
 from dvc.exceptions import DvcException
 
 
@@ -15,15 +17,17 @@ class DependencyIsNotFileOrDirError(DvcException):
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__(f"Stage file '{path}' cannot be a dependency.")
+        super().__init__(f"DVC file '{path}' cannot be a dependency.")
 
 
 class BaseDependency:
     IS_DEPENDENCY = True
 
-    DoesNotExistError = DependencyDoesNotExistError
-    IsNotFileOrDirError = DependencyIsNotFileOrDirError
-    IsStageFileError = DependencyIsStageFileError
+    DoesNotExistError = DependencyDoesNotExistError  # type: Type[DvcException]
+    IsNotFileOrDirError = (
+        DependencyIsNotFileOrDirError
+    )  # type: Type[DvcException]
+    IsStageFileError = DependencyIsStageFileError  # type: Type[DvcException]
 
     def update(self, rev=None):
         pass

dvc/dependency/gs.py

@@ -1,6 +1,8 @@
 from dvc.dependency.base import BaseDependency
-from dvc.output.gs import GSOutput
+from dvc.output.base import BaseOutput
+
+from ..fs.gs import GSFileSystem
 
 
-class GSDependency(BaseDependency, GSOutput):
-    pass
+class GSDependency(BaseDependency, BaseOutput):
+    FS_CLS = GSFileSystem

dvc/dependency/http.py

@@ -1,8 +1,8 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
 
-from ..tree.http import HTTPTree
+from ..fs.http import HTTPFileSystem
 
 
 class HTTPDependency(BaseDependency, BaseOutput):
-    TREE_CLS = HTTPTree
+    FS_CLS = HTTPFileSystem

dvc/dependency/https.py

@@ -1,6 +1,6 @@
-from ..tree.https import HTTPSTree
+from ..fs.https import HTTPSFileSystem
 from .http import HTTPDependency
 
 
 class HTTPSDependency(HTTPDependency):
-    TREE_CLS = HTTPSTree
+    FS_CLS = HTTPSFileSystem

dvc/dependency/param.py

@@ -2,12 +2,12 @@ import os
 from collections import defaultdict
 
 import dpath.util
-import toml
-import yaml
 from voluptuous import Any
 
 from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
+from dvc.hash_info import HashInfo
+from dvc.utils.serialize import LOADERS, ParseError
 
 
 class MissingParamsError(DvcException):
@@ -22,8 +22,6 @@ class ParamsDependency(LocalDependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
-    PARAMS_FILE_LOADERS = defaultdict(lambda: yaml.safe_load)
-    PARAMS_FILE_LOADERS.update({".toml": toml.load})
 
     def __init__(self, stage, path, params):
         info = {}
@@ -34,7 +32,7 @@ class ParamsDependency(LocalDependency):
             else:
                 assert isinstance(params, dict)
                 self.params = list(params.keys())
-                info = params
+                info = {self.PARAM_PARAMS: params}
 
         super().__init__(
             stage,
@@ -43,60 +41,61 @@ class ParamsDependency(LocalDependency):
             info=info,
         )
 
+    def dumpd(self):
+        ret = super().dumpd()
+        if not self.hash_info:
+            ret[self.PARAM_PARAMS] = self.params
+        return ret
+
     def fill_values(self, values=None):
         """Load params values dynamically."""
         if not values:
             return
+        info = {}
         for param in self.params:
             if param in values:
-                self.info[param] = values[param]
+                info[param] = values[param]
+        self.hash_info = HashInfo(self.PARAM_PARAMS, info)
 
-    def save(self):
-        super().save()
-        self.info = self.save_info()
+    def workspace_status(self):
+        status = super().workspace_status()
 
-    def status(self):
-        status = super().status()
-
-        if status[str(self)] == "deleted":
+        if status.get(str(self)) == "deleted":
             return status
 
         status = defaultdict(dict)
-        info = self.read_params()
+        info = self.hash_info.value if self.hash_info else {}
+        actual = self.read_params()
         for param in self.params:
-            if param not in info.keys():
+            if param not in actual.keys():
                 st = "deleted"
-            elif param not in self.info:
+            elif param not in info:
                 st = "new"
-            elif info[param] != self.info[param]:
+            elif actual[param] != info[param]:
                 st = "modified"
             else:
-                assert info[param] == self.info[param]
+                assert actual[param] == info[param]
                 continue
 
             status[str(self)][param] = st
 
         return status
 
-    def dumpd(self):
-        return {
-            self.PARAM_PATH: self.def_path,
-            self.PARAM_PARAMS: self.info or self.params,
-        }
+    def status(self):
+        return self.workspace_status()
 
     def read_params(self):
         if not self.exists:
             return {}
 
-        with self.repo.tree.open(self.path_info, "r") as fobj:
-            try:
-                config = self.PARAMS_FILE_LOADERS[
-                    self.path_info.suffix.lower()
-                ](fobj)
-            except (yaml.YAMLError, toml.TomlDecodeError) as exc:
-                raise BadParamFileError(
-                    f"Unable to read parameters from '{self}'"
-                ) from exc
+        suffix = self.path_info.suffix.lower()
+        loader = LOADERS[suffix]
+        try:
+            config = loader(self.path_info, fs=self.repo.fs)
+        except ParseError as exc:
+            raise BadParamFileError(
+                f"Unable to read parameters from '{self}'"
+            ) from exc
 
         ret = {}
         for param in self.params:
@@ -106,7 +105,7 @@ class ParamsDependency(LocalDependency):
                 pass
         return ret
 
-    def save_info(self):
+    def get_hash(self):
         info = self.read_params()
 
         missing_params = set(self.params) - set(info.keys())
@@ -117,4 +116,4 @@ class ParamsDependency(LocalDependency):
                 )
             )
 
-        return info
+        return HashInfo(self.PARAM_PARAMS, info)

dvc/dependency/repo.py

@@ -1,8 +1,5 @@
-import os
-
 from voluptuous import Required
 
-from dvc.exceptions import OutputNotFoundError
 from dvc.path_info import PathInfo
 
 from .local import LocalDependency
@@ -26,7 +23,7 @@ class RepoDependency(LocalDependency):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, tree, path):
+    def _parse_path(self, fs, path):
         return None
 
     @property
@@ -42,55 +39,72 @@ class RepoDependency(LocalDependency):
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    def _make_repo(self, *, locked=True):
+    def _make_repo(self, *, locked=True, **kwargs):
         from dvc.external_repo import external_repo
 
         d = self.def_repo
         rev = (d.get("rev_lock") if locked else None) or d.get("rev")
-        return external_repo(d["url"], rev=rev)
+        return external_repo(d["url"], rev=rev, **kwargs)
 
-    def _get_checksum(self, locked=True):
-        from dvc.repo.tree import RepoTree
+    def _get_hash(self, locked=True):
+        from dvc.objects.stage import stage
 
         with self._make_repo(locked=locked) as repo:
-            try:
-                return repo.find_out_by_relpath(self.def_path).info["md5"]
-            except OutputNotFoundError:
-                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+            path_info = PathInfo(repo.root_dir) / self.def_path
+            return stage(
+                self.repo.odb.local,
+                path_info,
+                repo.repo_fs,
+                self.repo.odb.local.fs.PARAM_CHECKSUM,
+                follow_subrepos=False,
+            ).hash_info
 
-                # we want stream but not fetch, so DVC out directories are
-                # walked, but dir contents is not fetched
-                tree = RepoTree(repo, stream=True)
+    def workspace_status(self):
+        current = self._get_hash(locked=True)
+        updated = self._get_hash(locked=False)
 
-                # We are polluting our repo cache with some dir listing here
-                if tree.isdir(path):
-                    return self.repo.cache.local.tree.get_hash(path, tree=tree)
-                return tree.get_file_hash(path)
-
-    def status(self):
-        current_checksum = self._get_checksum(locked=True)
-        updated_checksum = self._get_checksum(locked=False)
-
-        if current_checksum != updated_checksum:
+        if current != updated:
             return {str(self): "update available"}
 
         return {}
 
+    def status(self):
+        return self.workspace_status()
+
     def save(self):
         pass
 
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def download(self, to):
-        with self._make_repo() as repo:
+    def download(self, to, jobs=None):
+        from dvc.checkout import checkout
+        from dvc.config import NoRemoteError
+        from dvc.exceptions import NoOutputOrStageError
+        from dvc.objects import save
+        from dvc.objects.stage import stage
+
+        odb = self.repo.odb.local
+
+        with self._make_repo(cache_dir=odb.cache_dir) as repo:
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
                 self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
+            path_info = PathInfo(repo.root_dir) / self.def_path
+            try:
+                repo.fetch([path_info.fspath], jobs=jobs, recursive=True)
+            except (NoOutputOrStageError, NoRemoteError):
+                pass
+            obj = stage(
+                odb,
+                path_info,
+                repo.repo_fs,
+                odb.fs.PARAM_CHECKSUM,
+                jobs=jobs,
+                follow_subrepos=False,
+            )
+            save(odb, obj, jobs=jobs)
 
-            cache = self.repo.cache.local
-            with repo.use_cache(cache):
-                _, _, cache_infos = repo.fetch_external([self.def_path])
-            cache.checkout(to.path_info, cache_infos[0])
+        checkout(to.path_info, to.fs, obj, odb)
 
     def update(self, rev=None):
         if rev:

dvc/dependency/webdav.py

@@ -0,0 +1,8 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+
+from ..fs.webdav import WebDAVFileSystem
+
+
+class WebDAVDependency(BaseDependency, BaseOutput):
+    FS_CLS = WebDAVFileSystem

dvc/dependency/webdavs.py

@@ -0,0 +1,6 @@
+from ..fs.webdavs import WebDAVSFileSystem
+from .webdav import WebDAVDependency
+
+
+class WebDAVSDependency(WebDAVDependency):
+    FS_CLS = WebDAVSFileSystem

dvc/dependency/webhdfs.py

@@ -0,0 +1,6 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.webhdfs import WebHDFSOutput
+
+
+class WebHDFSDependency(BaseDependency, WebHDFSOutput):
+    pass

dvc/dvcfile.py

@@ -2,10 +2,12 @@ import collections
 import contextlib
 import logging
 import os
+from typing import TYPE_CHECKING, Any, Union
 
 from voluptuous import MultipleInvalid
 
 from dvc.exceptions import DvcException
+from dvc.parsing.versions import LOCKFILE_VERSION, SCHEMA_KWD
 from dvc.stage import serialize
 from dvc.stage.exceptions import (
     StageFileBadNameError,
@@ -13,10 +15,18 @@ from dvc.stage.exceptions import (
     StageFileFormatError,
     StageFileIsNotDvcFileError,
 )
-from dvc.stage.loader import SingleStageLoader, StageLoader
+from dvc.types import AnyPath
 from dvc.utils import relpath
 from dvc.utils.collections import apply_diff
-from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update
+from dvc.utils.serialize import (
+    dump_yaml,
+    modify_yaml,
+    parse_yaml,
+    parse_yaml_for_update,
+)
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
 
 logger = logging.getLogger(__name__)
 
@@ -26,10 +36,19 @@ PIPELINE_FILE = "dvc.yaml"
 PIPELINE_LOCK = "dvc.lock"
 
 
+class FileIsGitIgnored(DvcException):
+    def __init__(self, path):
+        super().__init__(f"'{path}' is git-ignored.")
+
+
 class LockfileCorruptedError(DvcException):
     pass
 
 
+class ParametrizedDumpError(DvcException):
+    pass
+
+
 def is_valid_filename(path):
     return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
         DVC_FILE,
@@ -39,16 +58,20 @@ def is_valid_filename(path):
 
 def is_dvc_file(path):
     return os.path.isfile(path) and (
-        is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
+        is_valid_filename(path) or is_lock_file(path)
     )
 
 
+def is_lock_file(path):
+    return os.path.basename(path) == PIPELINE_LOCK
+
+
 def check_dvc_filename(path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
-            "bad DVC-file name '{}'. DVC-files should be named "
-            "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
-                relpath(path), os.path.basename(path)
+            "bad DVC file name '{}'. DVC files should be named "
+            "'{}' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
+                relpath(path), PIPELINE_FILE, os.path.basename(path)
             )
         )
 
@@ -56,9 +79,10 @@ def check_dvc_filename(path):
 class FileMixin:
     SCHEMA = None
 
-    def __init__(self, repo, path, **kwargs):
+    def __init__(self, repo, path, verify=True, **kwargs):
         self.repo = repo
         self.path = path
+        self.verify = verify
 
     def __repr__(self):
         return "{}: {}".format(
@@ -81,30 +105,48 @@ class FileMixin:
         return relpath(self.path)
 
     def exists(self):
-        return self.repo.tree.exists(self.path)
+        return self.repo.fs.exists(self.path)
+
+    def _is_git_ignored(self):
+        from dvc.fs.local import LocalFileSystem
+
+        return isinstance(
+            self.repo.fs, LocalFileSystem
+        ) and self.repo.scm.is_ignored(self.path)
+
+    def _verify_filename(self):
+        if self.verify:
+            check_dvc_filename(self.path)
+
+    def _check_gitignored(self):
+        if self._is_git_ignored():
+            raise FileIsGitIgnored(self.path)
 
     def _load(self):
         # it raises the proper exceptions by priority:
         # 1. when the file doesn't exists
-        # 2. filename is not a DVC-file
+        # 2. filename is not a DVC file
         # 3. path doesn't represent a regular file
+        # 4. when the file is git ignored
         if not self.exists():
-            raise StageFileDoesNotExistError(self.path)
-        check_dvc_filename(self.path)
-        if not self.repo.tree.isfile(self.path):
+            is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
+            raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
+
+        self._verify_filename()
+        if not self.repo.fs.isfile(self.path):
             raise StageFileIsNotDvcFileError(self.path)
 
-        with self.repo.tree.open(self.path) as fd:
+        self._check_gitignored()
+        with self.repo.fs.open(self.path, encoding="utf-8") as fd:
             stage_text = fd.read()
         d = parse_yaml(stage_text, self.path)
-        self.validate(d, self.relpath)
-        return d, stage_text
+        return self.validate(d, self.relpath), stage_text
 
     @classmethod
     def validate(cls, d, fname=None):
         assert isinstance(cls.SCHEMA, collections.abc.Callable)
         try:
-            cls.SCHEMA(d)  # pylint: disable=not-callable
+            return cls.SCHEMA(d)  # pylint: disable=not-callable
         except MultipleInvalid as exc:
             raise StageFileFormatError(f"'{fname}' format error: {exc}")
 
@@ -115,26 +157,31 @@ class FileMixin:
     def dump(self, stage, **kwargs):
         raise NotImplementedError
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class SingleStageFile(FileMixin):
     from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
+    from dvc.stage.loader import SingleStageLoader as LOADER
 
     @property
     def stage(self):
         data, raw = self._load()
-        return SingleStageLoader.load_stage(self, data, raw)
+        return self.LOADER.load_stage(self, data, raw)
 
     @property
     def stages(self):
         data, raw = self._load()
-        return SingleStageLoader(self, data, raw)
+        return self.LOADER(self, data, raw)
 
     def dump(self, stage, **kwargs):
         """Dumps given stage appropriately in the dvcfile."""
         from dvc.stage import PipelineStage
 
         assert not isinstance(stage, PipelineStage)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
         logger.debug(
             "Saving information to '{file}'.".format(file=relpath(self.path))
         )
@@ -144,59 +191,70 @@ class SingleStageFile(FileMixin):
     def remove_stage(self, stage):  # pylint: disable=unused-argument
         self.remove()
 
+    def merge(self, ancestor, other):
+        assert isinstance(ancestor, SingleStageFile)
+        assert isinstance(other, SingleStageFile)
+
+        stage = self.stage
+        stage.merge(ancestor.stage, other.stage)
+        self.dump(stage)
+
 
 class PipelineFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
 
     from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
+    from dvc.stage.loader import StageLoader as LOADER
 
     @property
     def _lockfile(self):
         return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
 
     def dump(
-        self, stage, update_pipeline=False, no_lock=False, **kwargs
+        self, stage, update_pipeline=True, update_lock=True, **kwargs
     ):  # pylint: disable=arguments-differ
         """Dumps given stage appropriately in the dvcfile."""
         from dvc.stage import PipelineStage
 
         assert isinstance(stage, PipelineStage)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
 
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)
 
-        if not no_lock:
+        if update_lock:
             self._dump_lockfile(stage)
 
     def _dump_lockfile(self, stage):
         self._lockfile.dump(stage)
 
+    @staticmethod
+    def _check_if_parametrized(stage):
+        if stage.raw_data.parametrized:
+            raise ParametrizedDumpError(f"cannot dump a parametrized {stage}")
+
     def _dump_pipeline_file(self, stage):
-        data = {}
-        if self.exists():
-            with open(self.path) as fd:
-                data = parse_yaml_for_update(fd.read(), self.path)
-        else:
-            logger.info("Creating '%s'", self.relpath)
-            open(self.path, "w+").close()
-
-        data["stages"] = data.get("stages", {})
+        self._check_if_parametrized(stage)
         stage_data = serialize.to_pipeline_file(stage)
-        existing_entry = stage.name in data["stages"]
 
-        action = "Modifying" if existing_entry else "Adding"
-        logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
+        with modify_yaml(self.path, fs=self.repo.fs) as data:
+            if not data:
+                logger.info("Creating '%s'", self.relpath)
 
-        if existing_entry:
-            orig_stage_data = data["stages"][stage.name]
-            if "meta" in orig_stage_data:
-                stage_data[stage.name]["meta"] = orig_stage_data["meta"]
-            apply_diff(stage_data[stage.name], orig_stage_data)
-        else:
-            data["stages"].update(stage_data)
+            data["stages"] = data.get("stages", {})
+            existing_entry = stage.name in data["stages"]
+            action = "Modifying" if existing_entry else "Adding"
+            logger.info(
+                "%s stage '%s' in '%s'", action, stage.name, self.relpath
+            )
+
+            if existing_entry:
+                orig_stage_data = data["stages"][stage.name]
+                apply_diff(stage_data[stage.name], orig_stage_data)
+            else:
+                data["stages"].update(stage_data)
 
-        dump_yaml(self.path, data)
         self.repo.scm.track_file(self.relpath)
 
     @property
@@ -209,7 +267,7 @@ class PipelineFile(FileMixin):
     def stages(self):
         data, _ = self._load()
         lockfile_data = self._lockfile.load()
-        return StageLoader(self, data.get("stages", {}), lockfile_data)
+        return self.LOADER(self, data, lockfile_data)
 
     def remove(self, force=False):
         if not force:
@@ -224,7 +282,7 @@ class PipelineFile(FileMixin):
         if not self.exists():
             return
 
-        with open(self.path, "r") as f:
+        with open(self.path, "r", encoding="utf-8") as f:
             d = parse_yaml_for_update(f.read(), self.path)
 
         self.validate(d, self.path)
@@ -239,40 +297,95 @@ class PipelineFile(FileMixin):
         else:
             super().remove()
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
+
+def get_lockfile_schema(d):
+    from dvc.schema import (
+        COMPILED_LOCKFILE_V1_SCHEMA,
+        COMPILED_LOCKFILE_V2_SCHEMA,
+    )
+
+    schema = {
+        LOCKFILE_VERSION.V1: COMPILED_LOCKFILE_V1_SCHEMA,
+        LOCKFILE_VERSION.V2: COMPILED_LOCKFILE_V2_SCHEMA,
+    }
+
+    version = LOCKFILE_VERSION.from_dict(d)
+    return schema[version]
+
+
+def migrate_lock_v1_to_v2(d, version_info):
+    stages = {k: v for k, v in d.items()}
+
+    for key in stages:
+        d.pop(key)
+
+    # forcing order, meta should always be at the top
+    d.update(version_info)
+    d["stages"] = stages
+
 
 class Lockfile(FileMixin):
-    from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
-
-    def load(self):
-        if not self.exists():
-            return {}
-        with self.repo.tree.open(self.path) as fd:
-            data = parse_yaml(fd.read(), self.path)
+    @classmethod
+    def validate(cls, d, fname=None):
+        schema = get_lockfile_schema(d)
         try:
-            self.validate(data, fname=self.relpath)
-        except StageFileFormatError:
+            return schema(d)
+        except MultipleInvalid as exc:
+            raise StageFileFormatError(f"'{fname}' format error: {exc}")
+
+    def _verify_filename(self):
+        pass  # lockfile path is hardcoded, so no need to verify here
+
+    def _load(self):
+        try:
+            return super()._load()
+        except StageFileDoesNotExistError:
+            # we still need to account for git-ignored dvc.lock file
+            # even though it may not exist or have been .dvcignored
+            self._check_gitignored()
+            return {}, ""
+        except StageFileFormatError as exc:
             raise LockfileCorruptedError(
                 f"Lockfile '{self.relpath}' is corrupted."
-            )
-        return data
+            ) from exc
+
+    def load(self):
+        d, _ = self._load()
+        return d
+
+    @property
+    def latest_version_info(self):
+        version = LOCKFILE_VERSION.V2.value  # pylint:disable=no-member
+        return {SCHEMA_KWD: version}
 
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage)
-        if not self.exists():
-            modified = True
-            logger.info("Generating lock file '%s'", self.relpath)
-            data = stage_data
-            open(self.path, "w+").close()
-        else:
-            with self.repo.tree.open(self.path, "r") as fd:
-                data = parse_yaml_for_update(fd.read(), self.path)
-            modified = data.get(stage.name, {}) != stage_data.get(
+
+        with modify_yaml(self.path, fs=self.repo.fs) as data:
+            version = LOCKFILE_VERSION.from_dict(data)
+            if version == LOCKFILE_VERSION.V1:
+                logger.info(
+                    "Migrating lock file '%s' from v1 to v2", self.relpath
+                )
+                migrate_lock_v1_to_v2(data, self.latest_version_info)
+            else:
+                if not data:
+                    data.update(self.latest_version_info)
+                    # order is important, meta should always be at the top
+                    logger.info("Generating lock file '%s'", self.relpath)
+
+            data["stages"] = data.get("stages", {})
+            modified = data["stages"].get(stage.name, {}) != stage_data.get(
                 stage.name, {}
             )
             if modified:
                 logger.info("Updating lock file '%s'", self.relpath)
-            data.update(stage_data)
-        dump_yaml(self.path, data)
+
+            data["stages"].update(stage_data)
+
         if modified:
             self.repo.scm.track_file(self.relpath)
 
@@ -280,29 +393,41 @@ class Lockfile(FileMixin):
         if not self.exists():
             return
 
-        with open(self.path) as f:
+        with open(self.path, encoding="utf-8") as f:
             d = parse_yaml_for_update(f.read(), self.path)
         self.validate(d, self.path)
 
-        if stage.name not in d:
+        version = LOCKFILE_VERSION.from_dict(d)
+        data = d if version == LOCKFILE_VERSION.V1 else d.get("stages", {})
+        if stage.name not in data:
             return
 
         logger.debug("Removing '%s' from '%s'", stage.name, self.path)
-        del d[stage.name]
+        del data[stage.name]
 
-        if d:
+        if data:
             dump_yaml(self.path, d)
         else:
             self.remove()
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class Dvcfile:
-    def __new__(cls, repo, path, **kwargs):
+    def __new__(cls, repo: "Repo", path: AnyPath, **kwargs: Any):
         assert path
         assert repo
 
-        _, ext = os.path.splitext(path)
-        if ext in [".yaml", ".yml"]:
-            return PipelineFile(repo, path, **kwargs)
-        # fallback to single stage file for better error messages
-        return SingleStageFile(repo, path, **kwargs)
+        return make_dvcfile(repo, path, **kwargs)
+
+
+DVCFile = Union["PipelineFile", "SingleStageFile"]
+
+
+def make_dvcfile(repo: "Repo", path: AnyPath, **kwargs: Any) -> DVCFile:
+    _, ext = os.path.splitext(str(path))
+    if ext in [".yaml", ".yml"]:
+        return PipelineFile(repo, path, **kwargs)
+    # fallback to single stage file for better error messages
+    return SingleStageFile(repo, path, **kwargs)

dvc/env.py

@@ -1,2 +1,8 @@
+DVC_CHECKPOINT = "DVC_CHECKPOINT"
 DVC_DAEMON = "DVC_DAEMON"
 DVC_PAGER = "DVC_PAGER"
+DVC_ROOT = "DVC_ROOT"
+DVCLIVE_PATH = "DVCLIVE_PATH"
+DVCLIVE_SUMMARY = "DVCLIVE_SUMMARY"
+DVCLIVE_HTML = "DVCLIVE_HTML"
+DVCLIVE_RESUME = "DVCLIVE_RESUME"

dvc/exceptions.py

@@ -1,7 +1,5 @@
 """Exceptions raised by the dvc."""
-from funcy import first
-
-from dvc.utils import format_link, relpath
+from typing import List
 
 
 class DvcException(Exception):
@@ -9,6 +7,7 @@ class DvcException(Exception):
 
     def __init__(self, msg, *args):
         assert msg
+        self.msg = msg
         super().__init__(msg, *args)
 
 
@@ -26,6 +25,8 @@ class OutputDuplicationError(DvcException):
     """
 
     def __init__(self, output, stages):
+        from funcy import first
+
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
         if len(stages) == 1:
@@ -42,17 +43,19 @@ class OutputDuplicationError(DvcException):
 
 
 class OutputNotFoundError(DvcException):
-    """Thrown if a file/directory not found in repository pipelines.
+    """Thrown if a file/directory is not found as an output in any pipeline.
 
     Args:
         output (unicode): path to the file/directory.
     """
 
     def __init__(self, output, repo=None):
+        from dvc.utils import relpath
+
         self.output = output
         self.repo = repo
         super().__init__(
-            "Unable to find DVC-file with output '{path}'".format(
+            "Unable to find DVC file with output '{path}'".format(
                 path=relpath(self.output)
             )
         )
@@ -105,7 +108,7 @@ class ArgumentDuplicationError(DvcException):
 
 
 class MoveNotDataSourceError(DvcException):
-    """Thrown if attempted to move a file/directory that is not an output
+    """Thrown when trying to move a file/directory that is not an output
     in a data source stage.
 
     Args:
@@ -164,34 +167,40 @@ class BadMetricError(DvcException):
     def __init__(self, paths):
         super().__init__(
             "the following metrics do not exist, "
-            "are not metric files or are malformed: {paths}".format(
+            "are not metrics files or are malformed: {paths}".format(
                 paths=", ".join(f"'{path}'" for path in paths)
             )
         )
 
 
-class NoMetricsError(DvcException):
-    def __init__(self):
+class MetricsError(DvcException):
+    pass
+
+
+class NoMetricsParsedError(MetricsError):
+    def __init__(self, command):
         super().__init__(
-            "no metric files in this repository. Use `-m/-M` options for "
-            "`dvc run` to mark stage outputs as  metrics."
+            f"Could not parse {command} files. Use `-v` option to see more "
+            "details."
         )
 
 
-class NoPlotsError(DvcException):
-    def __init__(self):
+class NoMetricsFoundError(MetricsError):
+    def __init__(self, command, run_options):
         super().__init__(
-            "no plots in this repository. Use `--plots/--plots-no-cache` "
-            "options for `dvc run` to mark stage outputs as plots."
+            f"No {command} files in this repository. "
+            f"Use `{run_options}` options for "
+            f"`dvc run` to mark stage outputs as {command}."
         )
 
 
-class YAMLFileCorruptedError(DvcException):
-    def __init__(self, path):
-        path = relpath(path)
-        super().__init__(
-            f"unable to read: '{path}', YAML file structure is corrupted"
-        )
+class MetricDoesNotExistError(MetricsError):
+    def __init__(self, targets: List[str]):
+        if len(targets) == 1:
+            msg = "'{}' does not exist."
+        else:
+            msg = "'{}' do not exist."
+        super().__init__(msg.format(", ".join(targets)))
 
 
 class RecursiveAddingWhileUsingFilename(DvcException):
@@ -240,6 +249,8 @@ class DvcIgnoreInCollectedDirError(DvcException):
 
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
+        from dvc.utils import format_link
+
         super().__init__(
             "Hook '{}' already exists. Please refer to {} for more "
             "info.".format(
@@ -264,12 +275,16 @@ class UploadError(DvcException):
 
 class CheckoutError(DvcException):
     def __init__(self, target_infos, stats=None):
+        from dvc.utils import error_link
+
         self.target_infos = target_infos
         self.stats = stats
         targets = [str(t) for t in target_infos]
         m = (
-            "Checkout failed for following targets:\n{}\nDid you "
-            "forget to fetch?".format("\n".join(targets))
+            "Checkout failed for following targets:\n{}\nIs your "
+            "cache up to date?\n{}".format(
+                "\n".join(targets), error_link("missing-files"),
+            )
         )
         super().__init__(m)
 
@@ -287,6 +302,8 @@ class NoRemoteInExternalRepoError(DvcException):
 
 class NoOutputInExternalRepoError(DvcException):
     def __init__(self, path, external_repo_path, external_repo_url):
+        from dvc.utils import relpath
+
         super().__init__(
             "Output '{}' not found in target repository '{}'".format(
                 relpath(path, external_repo_path), external_repo_url
@@ -317,6 +334,8 @@ class PathMissingError(DvcException):
 
 class RemoteCacheRequiredError(DvcException):
     def __init__(self, path_info):
+        from dvc.utils import format_link
+
         super().__init__(
             (
                 "Current operation was unsuccessful because '{}' requires "
@@ -344,3 +363,24 @@ class NoOutputOrStageError(DvcException):
             f"'{target}' "
             f"does not exist as an output or a stage name in '{file}'"
         )
+
+
+class MergeError(DvcException):
+    pass
+
+
+class CacheLinkError(DvcException):
+    from dvc.utils import format_link
+
+    SUPPORT_LINK = "See {} for more information.".format(
+        format_link(
+            "https://dvc.org/doc/user-guide/troubleshooting#cache-types"
+        )
+    )
+
+    def __init__(self, path_infos):
+        msg = "No possible cache link types for '{}'. {}".format(
+            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK,
+        )
+        super().__init__(msg)
+        self.path_infos = path_infos

dvc/external_repo.py

@@ -3,62 +3,104 @@ import os
 import tempfile
 import threading
 from contextlib import contextmanager
-from distutils.dir_util import copy_tree
-from typing import Iterable
+from typing import Dict
 
-from funcy import cached_property, retry, wrap_with
+from funcy import retry, wrap_with
 
-from dvc.config import NoRemoteError, NotDvcRepoError
 from dvc.exceptions import (
     FileMissingError,
     NoOutputInExternalRepoError,
     NoRemoteInExternalRepoError,
+    NotDvcRepoError,
     OutputNotFoundError,
     PathMissingError,
 )
-from dvc.path_info import PathInfo
 from dvc.repo import Repo
-from dvc.repo.tree import RepoTree
-from dvc.scm.base import CloneError
-from dvc.scm.git import Git
-from dvc.tree.local import LocalTree
-from dvc.utils.fs import remove
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
 
 @contextmanager
-def external_repo(url, rev=None, for_write=False):
+def external_repo(
+    url, rev=None, for_write=False, cache_dir=None, cache_types=None, **kwargs
+):
+    from dvc.config import NoRemoteError
+    from dvc.scm.git import Git
+
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
-    if not rev:
-        # Local HEAD points to the tip of whatever branch we first cloned from
-        # (which may not be the default branch), use origin/HEAD here to get
-        # the tip of the default branch
-        rev = "refs/remotes/origin/HEAD"
-    try:
-        repo = ExternalRepo(path, url, rev, for_write=for_write)
-    except NotDvcRepoError:
-        repo = ExternalGitRepo(path, url, rev)
+    # Local HEAD points to the tip of whatever branch we first cloned from
+    # (which may not be the default branch), use origin/HEAD here to get
+    # the tip of the default branch
+    rev = rev or "refs/remotes/origin/HEAD"
+
+    cache_config = {
+        "cache": {
+            "dir": cache_dir or _get_cache_dir(url),
+            "type": cache_types,
+        }
+    }
+
+    config = _get_remote_config(url) if os.path.isdir(url) else {}
+    config.update(cache_config)
+
+    def make_repo(path, **_kwargs):
+        _config = cache_config.copy()
+        if os.path.isdir(url):
+            rel = os.path.relpath(path, _kwargs["scm"].root_dir)
+            repo_path = os.path.join(url, rel)
+            _config.update(_get_remote_config(repo_path))
+        return Repo(path, config=_config, **_kwargs)
+
+    root_dir = path if for_write else os.path.realpath(path)
+    repo_kwargs = dict(
+        root_dir=root_dir,
+        url=url,
+        scm=None if for_write else Git(root_dir),
+        rev=None if for_write else rev,
+        config=config,
+        repo_factory=make_repo,
+        **kwargs,
+    )
+
+    if "subrepos" not in repo_kwargs:
+        repo_kwargs["subrepos"] = True
+
+    if "uninitialized" not in repo_kwargs:
+        repo_kwargs["uninitialized"] = True
+
+    repo = Repo(**repo_kwargs)
 
     try:
         yield repo
-    except NoRemoteError:
-        raise NoRemoteInExternalRepoError(url)
+    except NoRemoteError as exc:
+        raise NoRemoteInExternalRepoError(url) from exc
     except OutputNotFoundError as exc:
         if exc.repo is repo:
-            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
+            raise NoOutputInExternalRepoError(
+                exc.output, repo.root_dir, url
+            ) from exc
         raise
     except FileMissingError as exc:
-        raise PathMissingError(exc.path, url)
+        raise PathMissingError(exc.path, url) from exc
     finally:
         repo.close()
         if for_write:
             _remove(path)
 
 
-CLONES = {}
-CACHE_DIRS = {}
+CLONES: Dict[str, str] = {}
+CACHE_DIRS: Dict[str, str] = {}
+
+
+@wrap_with(threading.Lock())
+def _get_cache_dir(url):
+    try:
+        cache_dir = CACHE_DIRS[url]
+    except KeyError:
+        cache_dir = CACHE_DIRS[url] = tempfile.mkdtemp("dvc-cache")
+    return cache_dir
 
 
 def clean_repos():
@@ -71,179 +113,28 @@ def clean_repos():
         _remove(path)
 
 
-class BaseExternalRepo:
-    # pylint: disable=no-member
+def _get_remote_config(url):
+    try:
+        repo = Repo(url)
+    except NotDvcRepoError:
+        return {}
 
-    _local_cache = None
+    try:
+        name = repo.config["core"].get("remote")
+        if not name:
+            # Fill the empty upstream entry with a new remote pointing to the
+            # original repo's cache location.
+            name = "auto-generated-upstream"
+            return {
+                "core": {"remote": name},
+                "remote": {name: {"url": repo.odb.local.cache_dir}},
+            }
 
-    def __str__(self):
-        return self.url
-
-    @property
-    def local_cache(self):
-        if hasattr(self, "cache"):
-            return self.cache.local
-        return self._local_cache
-
-    @contextmanager
-    def use_cache(self, cache):
-        """Use the specified cache in place of default tmpdir cache for
-        download operations.
-        """
-        if hasattr(self, "cache"):
-            save_cache = self.cache.local
-            self.cache.local = cache
-        self._local_cache = cache
-
-        yield
-
-        if hasattr(self, "cache"):
-            self.cache.local = save_cache
-        self._local_cache = None
-
-    @cached_property
-    def repo_tree(self):
-        return RepoTree(self, fetch=True)
-
-    def get_rev(self):
-        if isinstance(self.tree, LocalTree):
-            return self.scm.get_rev()
-        return self.tree.rev
-
-    def fetch_external(self, paths: Iterable, **kwargs):
-        """Fetch specified external repo paths into cache.
-
-        Returns 3-tuple in the form
-            (downloaded, failed, list(cache_infos))
-        where cache_infos can be used as checkout targets for the
-        fetched paths.
-        """
-        download_results = []
-        failed = 0
-
-        paths = [PathInfo(self.root_dir) / path for path in paths]
-
-        def download_update(result):
-            download_results.append(result)
-
-        save_infos = []
-        for path in paths:
-            if not self.repo_tree.exists(path):
-                raise PathMissingError(path, self.url)
-            save_info = self.local_cache.save(
-                path,
-                self.repo_tree,
-                None,
-                save_link=False,
-                download_callback=download_update,
-            )
-            save_infos.append(save_info)
-
-        return sum(download_results), failed, save_infos
-
-    def get_external(self, path, dest):
-        """Convenience wrapper for fetch_external and checkout."""
-        if self.local_cache:
-            # fetch DVC and git files to tmpdir cache, then checkout
-            _, _, save_infos = self.fetch_external([path])
-            self.local_cache.checkout(PathInfo(dest), save_infos[0])
-        else:
-            # git-only erepo with no cache, just copy files directly
-            # to dest
-            path = PathInfo(self.root_dir) / path
-            if not self.repo_tree.exists(path):
-                raise PathMissingError(path, self.url)
-            self.repo_tree.copytree(path, dest)
-
-
-class ExternalRepo(Repo, BaseExternalRepo):
-    def __init__(self, root_dir, url, rev, for_write=False):
-        if for_write:
-            super().__init__(root_dir)
-        else:
-            root_dir = os.path.realpath(root_dir)
-            super().__init__(root_dir, scm=Git(root_dir), rev=rev)
-        self.url = url
-        self._set_cache_dir()
-        self._fix_upstream()
-
-    @wrap_with(threading.Lock())
-    def _set_cache_dir(self):
-        try:
-            cache_dir = CACHE_DIRS[self.url]
-        except KeyError:
-            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
-
-        self.cache.local.cache_dir = cache_dir
-        self._local_cache = self.cache.local
-
-    def _fix_upstream(self):
-        if not os.path.isdir(self.url):
-            return
-
-        try:
-            src_repo = Repo(self.url)
-        except NotDvcRepoError:
-            # If ExternalRepo does not throw NotDvcRepoError and Repo does,
-            # the self.url might be a bare git repo.
-            # NOTE: This will fail to resolve remote with relative path,
-            # same as if it was a remote DVC repo.
-            return
-
-        try:
-            remote_name = self.config["core"].get("remote")
-            if remote_name:
-                self._fix_local_remote(src_repo, remote_name)
-            else:
-                self._add_upstream(src_repo)
-        finally:
-            src_repo.close()
-
-    def _fix_local_remote(self, src_repo, remote_name):
-        # If a remote URL is relative to the source repo,
-        # it will have changed upon config load and made
-        # relative to this new repo. Restore the old one here.
-        new_remote = self.config["remote"][remote_name]
-        old_remote = src_repo.config["remote"][remote_name]
-        if new_remote["url"] != old_remote["url"]:
-            new_remote["url"] = old_remote["url"]
-
-    def _add_upstream(self, src_repo):
-        # Fill the empty upstream entry with a new remote pointing to the
-        # original repo's cache location.
-        cache_dir = src_repo.cache.local.cache_dir
-        self.config["remote"]["auto-generated-upstream"] = {"url": cache_dir}
-        self.config["core"]["remote"] = "auto-generated-upstream"
-
-
-class ExternalGitRepo(BaseExternalRepo):
-    def __init__(self, root_dir, url, rev):
-        self.root_dir = os.path.realpath(root_dir)
-        self.url = url
-        self.tree = self.scm.get_tree(rev)
-
-    @cached_property
-    def scm(self):
-        return Git(self.root_dir)
-
-    def close(self):
-        if "scm" in self.__dict__:
-            self.scm.close()
-
-    def find_out_by_relpath(self, path):
-        raise OutputNotFoundError(path, self)
-
-    @contextmanager
-    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
-        """Opens a specified resource as a file object."""
-        tree = RepoTree(self)
-        try:
-            with tree.open(
-                path, mode=mode, encoding=encoding, **kwargs
-            ) as fobj:
-                yield fobj
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
+        # Use original remote to make sure that we are using correct url,
+        # credential paths, etc if they are relative to the config location.
+        return {"remote": {name: repo.config["remote"][name]}}
+    finally:
+        repo.close()
 
 
 def _cached_clone(url, rev, for_write=False):
@@ -253,6 +144,8 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
+    from distutils.dir_util import copy_tree
+
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
     clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
@@ -279,6 +172,8 @@ def _clone_default_branch(url, rev, for_write=False):
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
+    from dvc.scm.git import Git
+
     clone_path, shallow = CLONES.get(url, (None, False))
 
     git = None
@@ -306,6 +201,8 @@ def _clone_default_branch(url, rev, for_write=False):
             clone_path = tempfile.mkdtemp("dvc-clone")
             if not for_write and rev and not Git.is_sha(rev):
                 # If rev is a tag or branch name try shallow clone first
+                from dvc.scm.base import CloneError
+
                 try:
                     git = Git.clone(url, clone_path, shallow_branch=rev)
                     shallow = True
@@ -326,19 +223,21 @@ def _clone_default_branch(url, rev, for_write=False):
 
 
 def _unshallow(git):
-    if git.repo.head.is_detached:
+    if git.gitpython.repo.head.is_detached:
         # If this is a detached head (i.e. we shallow cloned a tag) switch to
         # the default branch
-        origin_refs = git.repo.remotes["origin"].refs
+        origin_refs = git.gitpython.repo.remotes["origin"].refs
         ref = origin_refs["HEAD"].reference
         branch_name = ref.name.split("/")[-1]
-        branch = git.repo.create_head(branch_name, ref)
+        branch = git.gitpython.repo.create_head(branch_name, ref)
         branch.set_tracking_branch(ref)
         branch.checkout()
     git.pull(unshallow=True)
 
 
 def _git_checkout(repo_path, rev):
+    from dvc.scm.git import Git
+
     logger.debug("erepo: git checkout %s@%s", repo_path, rev)
     git = Git(repo_path)
     try:
@@ -348,9 +247,16 @@ def _git_checkout(repo_path, rev):
 
 
 def _remove(path):
+    from dvc.utils.fs import remove
+
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(path)
+        try:
+            os_retry(remove)(path)
+        except PermissionError:
+            logger.warning(
+                "Failed to remove '%s'", relpath(path), exc_info=True
+            )
     else:
         remove(path)

dvc/fs/__init__.py

@@ -0,0 +1,90 @@
+import posixpath
+from urllib.parse import urlparse
+
+from ..scheme import Schemes
+from .azure import AzureFileSystem
+from .gdrive import GDriveFileSystem
+from .gs import GSFileSystem
+from .hdfs import HDFSFileSystem
+from .http import HTTPFileSystem
+from .https import HTTPSFileSystem
+from .local import LocalFileSystem
+from .oss import OSSFileSystem
+from .s3 import S3FileSystem
+from .ssh import SSHFileSystem
+from .webdav import WebDAVFileSystem
+from .webdavs import WebDAVSFileSystem
+from .webhdfs import WebHDFSFileSystem
+
+FS_MAP = {
+    Schemes.AZURE: AzureFileSystem,
+    Schemes.GDRIVE: GDriveFileSystem,
+    Schemes.GS: GSFileSystem,
+    Schemes.HDFS: HDFSFileSystem,
+    Schemes.WEBHDFS: WebHDFSFileSystem,
+    Schemes.HTTP: HTTPFileSystem,
+    Schemes.HTTPS: HTTPSFileSystem,
+    Schemes.S3: S3FileSystem,
+    Schemes.SSH: SSHFileSystem,
+    Schemes.OSS: OSSFileSystem,
+    Schemes.WEBDAV: WebDAVFileSystem,
+    Schemes.WEBDAVS: WebDAVSFileSystem,
+    # NOTE: LocalFileSystem is the default
+}
+
+
+def get_fs_cls(remote_conf):
+    scheme = urlparse(remote_conf["url"]).scheme
+    return FS_MAP.get(scheme, LocalFileSystem)
+
+
+def get_fs_config(config, **kwargs):
+    name = kwargs.get("name")
+    if name:
+        remote_conf = config["remote"][name.lower()]
+    else:
+        remote_conf = kwargs
+    return _resolve_remote_refs(config, remote_conf)
+
+
+def _resolve_remote_refs(config, remote_conf):
+    # Support for cross referenced remotes.
+    # This will merge the settings, shadowing base ref with remote_conf.
+    # For example, having:
+    #
+    #       dvc remote add server ssh://localhost
+    #       dvc remote modify server user root
+    #       dvc remote modify server ask_password true
+    #
+    #       dvc remote add images remote://server/tmp/pictures
+    #       dvc remote modify images user alice
+    #       dvc remote modify images ask_password false
+    #       dvc remote modify images password asdf1234
+    #
+    # Results on a config dictionary like:
+    #
+    #       {
+    #           "url": "ssh://localhost/tmp/pictures",
+    #           "user": "alice",
+    #           "password": "asdf1234",
+    #           "ask_password": False,
+    #       }
+    parsed = urlparse(remote_conf["url"])
+    if parsed.scheme != "remote":
+        return remote_conf
+
+    base = get_fs_config(config, name=parsed.netloc)
+    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    return {**base, **remote_conf, "url": url}
+
+
+def get_cloud_fs(repo, **kwargs):
+    from dvc.config import ConfigError
+    from dvc.config_schema import SCHEMA, Invalid
+
+    remote_conf = get_fs_config(repo.config, **kwargs)
+    try:
+        remote_conf = SCHEMA["remote"][str](remote_conf)
+    except Invalid as exc:
+        raise ConfigError(str(exc)) from None
+    return get_fs_cls(remote_conf)(repo, remote_conf)

dvc/fs/_metadata.py

@@ -0,0 +1,72 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, List
+
+from dvc.output import BaseOutput
+from dvc.path_info import PathInfo
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+
+@dataclass
+class Metadata:
+    """
+    Container for storing metadata for a given path, similar in spirit to
+    `os.stat_result`.
+    """
+
+    # required field
+    path_info: PathInfo
+    repo: "Repo"
+
+    # computed fields
+    is_output: bool = field(init=False, default=False)  # is it an output?
+    # is the path part of an output?
+    part_of_output: bool = field(init=False, default=False)
+    # does the path contain outputs?
+    contains_outputs: bool = field(init=False, default=False)
+    # is the path tracked by dvc? equivalent to (is_output or part_of_output)
+    is_dvc: bool = field(init=False, default=False)
+    # does dvc has any output in that path or inside it? equiv. to bool(outs)
+    output_exists: bool = field(init=False)
+
+    # optional fields
+    isdir: bool = False  # is it a directory?
+    is_exec: bool = False  # is it an executable?
+    outs: List[BaseOutput] = field(default_factory=list)  # list of outputs
+
+    def __post_init__(self):
+        self.output_exists = bool(self.outs)
+
+        if not self.output_exists:
+            return
+
+        # it can contain multiple outputs
+        if len(self.outs) > 1:
+            self.contains_outputs = True
+        else:
+            out = self.outs[0]
+            # or, the path itself could be an output
+            self.is_output = self.path_info == out.path_info
+
+            # or, a directory must have an output somewhere deep inside it
+            if not self.is_output:
+                self.contains_outputs = out.path_info.isin(self.path_info)
+
+        # or, the path could be a part of an output, i.e. inside of an output
+        self.part_of_output = not self.is_output and not self.contains_outputs
+
+        self.is_dvc = self.is_output or self.part_of_output
+        # if it contains outputs, it must be a dir
+        self.isdir = self.contains_outputs
+
+    @property
+    def isfile(self):
+        # self.isdir might require additional computation, especially if it's
+        # a dir that's inside a dvc-tracked dir
+        # since, `Metadata()` does not support being used as null (i.e. when
+        # path does not exist), `isfile` becomes `isdir`'s complement.
+        return not self.isdir
+
+    def __str__(self):
+        return f"Metadata: {self.path_info}"

dvc/fs/azure.py

@@ -0,0 +1,125 @@
+import logging
+import os
+import threading
+
+from funcy import cached_property, wrap_prop
+
+from dvc.exceptions import DvcException
+from dvc.path_info import CloudURLInfo
+from dvc.scheme import Schemes
+from dvc.utils import format_link
+
+from .fsspec_wrapper import FSSpecWrapper
+
+logger = logging.getLogger(__name__)
+_DEFAULT_CREDS_STEPS = (
+    "https://azuresdkdocs.blob.core.windows.net/$web/python/"
+    "azure-identity/1.4.0/azure.identity.html#azure.identity"
+    ".DefaultAzureCredential"
+)
+
+
+class AzureAuthError(DvcException):
+    pass
+
+
+class AzureFileSystem(FSSpecWrapper):
+    scheme = Schemes.AZURE
+    PATH_CLS = CloudURLInfo
+    PARAM_CHECKSUM = "etag"
+    DETAIL_FIELDS = frozenset(("etag", "size"))
+    REQUIRES = {
+        "adlfs": "adlfs",
+        "knack": "knack",
+        "azure-identity": "azure.identity",
+    }
+
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+
+        url = config.get("url")
+        self.path_info = self.PATH_CLS(url)
+        self.bucket = self.path_info.bucket
+
+        if not self.bucket:
+            container = self._az_config.get("storage", "container_name", None)
+            url = f"azure://{container}"
+
+        self.path_info = self.PATH_CLS(url)
+        self.bucket = self.path_info.bucket
+
+    def _prepare_credentials(self, config):
+        from azure.identity.aio import DefaultAzureCredential
+
+        login_info = {}
+        login_info["connection_string"] = config.get(
+            "connection_string",
+            self._az_config.get("storage", "connection_string", None),
+        )
+        login_info["account_name"] = config.get(
+            "account_name", self._az_config.get("storage", "account", None)
+        )
+        login_info["account_key"] = config.get(
+            "account_key", self._az_config.get("storage", "key", None)
+        )
+        login_info["sas_token"] = config.get(
+            "sas_token", self._az_config.get("storage", "sas_token", None)
+        )
+        login_info["tenant_id"] = config.get("tenant_id")
+        login_info["client_id"] = config.get("client_id")
+        login_info["client_secret"] = config.get("client_secret")
+
+        if not any(login_info.values()):
+            login_info["credentials"] = DefaultAzureCredential()
+
+        for login_method, required_keys in [  # noqa
+            ("connection string", ["connection_string"]),
+            (
+                "AD service principal",
+                ["tenant_id", "client_id", "client_secret"],
+            ),
+            ("account key", ["account_name", "account_key"]),
+            ("SAS token", ["account_name", "sas_token"]),
+            ("anonymous login", ["account_name"]),
+            (f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
+        ]:
+            if all(login_info.get(key) is not None for key in required_keys):
+                break
+        else:
+            login_method = None
+
+        self.login_method = login_method
+        return login_info
+
+    @cached_property
+    def _az_config(self):
+        # NOTE: ideally we would've used get_default_cli().config from
+        # azure.cli.core, but azure-cli-core has a lot of conflicts with other
+        # dependencies. So instead we are just use knack directly
+        from knack.config import CLIConfig
+
+        config_dir = os.getenv(
+            "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
+        )
+        return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def fs(self):
+        from adlfs import AzureBlobFileSystem
+        from azure.core.exceptions import AzureError
+
+        try:
+            file_system = AzureBlobFileSystem(**self.fs_args)
+            if self.bucket not in [
+                container.rstrip("/") for container in file_system.ls("/")
+            ]:
+                file_system.mkdir(self.bucket)
+        except (ValueError, AzureError) as e:
+            raise AzureAuthError(
+                f"Authentication to Azure Blob Storage via {self.login_method}"
+                " failed.\nLearn more about configuration settings at"
+                f" {format_link('https://man.dvc.org/remote/modify')}"
+            ) from e
+
+        return file_system

dvc/fs/base.py

@@ -0,0 +1,338 @@
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial
+from multiprocessing import cpu_count
+from typing import Any, ClassVar, Dict, FrozenSet, Optional
+
+from funcy import cached_property
+
+from dvc.exceptions import DvcException
+from dvc.path_info import URLInfo
+from dvc.progress import Tqdm
+from dvc.utils import tmp_fname
+from dvc.utils.fs import makedirs, move
+from dvc.utils.http import open_url
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteCmdError(DvcException):
+    def __init__(self, remote, cmd, ret, err):
+        super().__init__(
+            "{remote} command '{cmd}' finished with non-zero return code"
+            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
+        )
+
+
+class RemoteActionNotImplemented(DvcException):
+    def __init__(self, action, scheme):
+        m = f"{action} is not supported for {scheme} remotes"
+        super().__init__(m)
+
+
+class RemoteMissingDepsError(DvcException):
+    pass
+
+
+class BaseFileSystem:
+    scheme = "base"
+    REQUIRES: ClassVar[Dict[str, str]] = {}
+    PATH_CLS = URLInfo  # type: Any
+    JOBS = 4 * cpu_count()
+
+    CHECKSUM_DIR_SUFFIX = ".dir"
+    HASH_JOBS = max(1, min(4, cpu_count() // 2))
+    LIST_OBJECT_PAGE_SIZE = 1000
+    TRAVERSE_WEIGHT_MULTIPLIER = 5
+    TRAVERSE_PREFIX_LEN = 3
+    TRAVERSE_THRESHOLD_SIZE = 500000
+    CAN_TRAVERSE = True
+
+    # Needed for some providers, and http open()
+    CHUNK_SIZE = 64 * 1024 * 1024  # 64 MiB
+
+    PARAM_CHECKSUM: ClassVar[Optional[str]] = None
+    DETAIL_FIELDS: FrozenSet[str] = frozenset()
+
+    def __init__(self, repo, config):
+        self.repo = repo
+        self.config = config
+
+        self._check_requires()
+
+        self.path_info = None
+
+    @cached_property
+    def jobs(self):
+        return (
+            self.config.get("jobs")
+            or (self.repo and self.repo.config["core"].get("jobs"))
+            or self.JOBS
+        )
+
+    @cached_property
+    def hash_jobs(self):
+        return (
+            self.config.get("checksum_jobs")
+            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
+            or self.HASH_JOBS
+        )
+
+    @classmethod
+    def get_missing_deps(cls):
+        import importlib
+
+        missing = []
+        for package, module in cls.REQUIRES.items():
+            try:
+                importlib.import_module(module)
+            except ImportError:
+                missing.append(package)
+
+        return missing
+
+    def _check_requires(self):
+        from ..scheme import Schemes
+        from ..utils import format_link
+        from ..utils.pkg import PKG
+
+        missing = self.get_missing_deps()
+        if not missing:
+            return
+
+        url = self.config.get("url", f"{self.scheme}://")
+
+        scheme = self.scheme
+        if scheme == Schemes.WEBDAVS:
+            scheme = Schemes.WEBDAV
+
+        by_pkg = {
+            "pip": f"pip install 'dvc[{scheme}]'",
+            "conda": f"conda install -c conda-forge dvc-{scheme}",
+        }
+
+        cmd = by_pkg.get(PKG)
+        if cmd:
+            link = format_link("https://dvc.org/doc/install")
+            hint = (
+                f"To install dvc with those dependencies, run:\n"
+                "\n"
+                f"\t{cmd}\n"
+                "\n"
+                f"See {link} for more info."
+            )
+        else:
+            link = format_link("https://github.com/iterative/dvc/issues")
+            hint = f"Please report this bug to {link}. Thank you!"
+
+        raise RemoteMissingDepsError(
+            f"URL '{url}' is supported but requires these missing "
+            f"dependencies: {missing}. {hint}"
+        )
+
+    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
+        if hasattr(self, "_generate_download_url"):
+            # pylint:disable=no-member
+            func = self._generate_download_url  # type: ignore[attr-defined]
+            get_url = partial(func, path_info)
+            return open_url(get_url, mode=mode, encoding=encoding, **kwargs)
+
+        raise RemoteActionNotImplemented("open", self.scheme)
+
+    def exists(self, path_info, use_dvcignore=True) -> bool:
+        raise NotImplementedError
+
+    # pylint: disable=unused-argument
+
+    def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return False
+
+    def isfile(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
+
+    def isexec(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between executable and non-executable file.
+        """
+        return False
+
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
+    def walk_files(self, path_info, **kwargs):
+        """Return a generator with `PathInfo`s to all the files.
+
+        Optional kwargs:
+            prefix (bool): If true `path_info` will be treated as a prefix
+                rather than directory path.
+        """
+        raise NotImplementedError
+
+    def ls(self, path_info, detail=False, **kwargs):
+        raise RemoteActionNotImplemented("ls", self.scheme)
+
+    def is_empty(self, path_info):
+        return False
+
+    def info(self, path_info):
+        raise NotImplementedError
+
+    def getsize(self, path_info):
+        return self.info(path_info).get("size")
+
+    def remove(self, path_info):
+        raise RemoteActionNotImplemented("remove", self.scheme)
+
+    def makedirs(self, path_info):
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+
+    def move(self, from_info, to_info):
+        self.copy(from_info, to_info)
+        self.remove(from_info)
+
+    def copy(self, from_info, to_info):
+        raise RemoteActionNotImplemented("copy", self.scheme)
+
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
+    # pylint: enable=unused-argument
+
+    @classmethod
+    def is_dir_hash(cls, hash_):
+        if not hash_:
+            return False
+        return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
+
+    def upload(
+        self, from_info, to_info, name=None, no_progress_bar=False,
+    ):
+        if not hasattr(self, "_upload"):
+            raise RemoteActionNotImplemented("upload", self.scheme)
+
+        if to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        if from_info.scheme != "local":
+            raise NotImplementedError
+
+        logger.debug("Uploading '%s' to '%s'", from_info, to_info)
+
+        name = name or from_info.name
+
+        self._upload(  # noqa, pylint: disable=no-member
+            from_info.fspath,
+            to_info,
+            name=name,
+            no_progress_bar=no_progress_bar,
+        )
+
+    def upload_fobj(self, fobj, to_info, no_progress_bar=False, **pbar_args):
+        if not hasattr(self, "_upload_fobj"):
+            raise RemoteActionNotImplemented("upload_fobj", self.scheme)
+
+        with Tqdm.wrapattr(
+            fobj, "read", disable=no_progress_bar, bytes=True, **pbar_args
+        ) as wrapped:
+            self._upload_fobj(wrapped, to_info)  # pylint: disable=no-member
+
+    def download(
+        self,
+        from_info,
+        to_info,
+        name=None,
+        no_progress_bar=False,
+        jobs=None,
+        **kwargs,
+    ):
+        if not hasattr(self, "_download"):
+            raise RemoteActionNotImplemented("download", self.scheme)
+
+        if from_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        if to_info.scheme == self.scheme != "local":
+            self.copy(from_info, to_info)
+            return 0
+
+        if to_info.scheme != "local":
+            raise NotImplementedError
+
+        if self.isdir(from_info):
+            return self._download_dir(
+                from_info, to_info, name, no_progress_bar, jobs, **kwargs,
+            )
+        return self._download_file(from_info, to_info, name, no_progress_bar,)
+
+    def _download_dir(
+        self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
+    ):
+        from_infos = list(self.walk_files(from_info, **kwargs))
+        if not from_infos:
+            makedirs(to_info, exist_ok=True)
+            return None
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
+        )
+
+        with Tqdm(
+            total=len(from_infos),
+            desc="Downloading directory",
+            unit="Files",
+            disable=no_progress_bar,
+        ) as pbar:
+            download_files = pbar.wrap_fn(
+                partial(self._download_file, name=name, no_progress_bar=True,)
+            )
+            max_workers = jobs or self.jobs
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                futures = [
+                    executor.submit(download_files, from_info, to_info)
+                    for from_info, to_info in zip(from_infos, to_infos)
+                ]
+
+                # NOTE: unlike pulling/fetching cache, where we need to
+                # download everything we can, not raising an error here might
+                # turn very ugly, as the user might think that he has
+                # downloaded a complete directory, while having a partial one,
+                # which might cause unexpected results in his pipeline.
+                for future in as_completed(futures):
+                    # NOTE: executor won't let us raise until all futures that
+                    # it has are finished, so we need to cancel them ourselves
+                    # before re-raising.
+                    exc = future.exception()
+                    if exc:
+                        for entry in futures:
+                            entry.cancel()
+                        raise exc
+
+    def _download_file(
+        self, from_info, to_info, name, no_progress_bar,
+    ):
+        makedirs(to_info.parent, exist_ok=True)
+
+        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
+        name = name or to_info.name
+
+        tmp_file = tmp_fname(to_info)
+
+        self._download(  # noqa, pylint: disable=no-member
+            from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
+        )
+
+        move(tmp_file, to_info)

dvc/fs/dvc.py

@@ -0,0 +1,242 @@
+import logging
+import os
+import typing
+
+from dvc.exceptions import OutputNotFoundError
+from dvc.path_info import PathInfo
+from dvc.utils import relpath
+
+from ._metadata import Metadata
+from .base import BaseFileSystem
+
+if typing.TYPE_CHECKING:
+    from dvc.output.base import BaseOutput
+
+
+logger = logging.getLogger(__name__)
+
+
+class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
+    """DVC repo fs.
+
+    Args:
+        repo: DVC repo.
+    """
+
+    scheme = "local"
+    PARAM_CHECKSUM = "md5"
+
+    def __init__(self, repo):
+        super().__init__(repo, {"url": repo.root_dir})
+
+    def _find_outs(self, path, *args, **kwargs):
+        outs = self.repo.find_outs_by_path(path, *args, **kwargs)
+
+        def _is_cached(out):
+            return out.use_cache
+
+        outs = list(filter(_is_cached, outs))
+        if not outs:
+            raise OutputNotFoundError(path, self.repo)
+
+        return outs
+
+    def _get_granular_hash(
+        self, path_info: PathInfo, out: "BaseOutput", remote=None
+    ):
+        assert isinstance(path_info, PathInfo)
+        # NOTE: use string paths here for performance reasons
+        key = tuple(relpath(path_info, out.path_info).split(os.sep))
+        out.get_dir_cache(remote=remote)
+        obj = out.obj.trie.get(key)
+        if obj:
+            return obj.hash_info
+        raise FileNotFoundError
+
+    def open(  # type: ignore
+        self, path: PathInfo, mode="r", encoding=None, remote=None, **kwargs
+    ):  # pylint: disable=arguments-differ
+        try:
+            outs = self._find_outs(path, strict=False)
+        except OutputNotFoundError as exc:
+            raise FileNotFoundError from exc
+
+        if len(outs) != 1 or (
+            outs[0].is_dir_checksum and path == outs[0].path_info
+        ):
+            raise IsADirectoryError
+
+        out = outs[0]
+        if out.changed_cache(filter_info=path):
+            from dvc.config import NoRemoteError
+
+            try:
+                remote_obj = self.repo.cloud.get_remote(remote)
+            except NoRemoteError:
+                raise FileNotFoundError
+            if out.is_dir_checksum:
+                checksum = self._get_granular_hash(path, out).value
+            else:
+                checksum = out.hash_info.value
+            remote_info = remote_obj.odb.hash_to_path_info(checksum)
+            return remote_obj.fs.open(
+                remote_info, mode=mode, encoding=encoding
+            )
+
+        if out.is_dir_checksum:
+            checksum = self._get_granular_hash(path, out).value
+            cache_path = out.odb.hash_to_path_info(checksum).url
+        else:
+            cache_path = out.cache_path
+        return open(cache_path, mode=mode, encoding=encoding)
+
+    def exists(self, path):  # pylint: disable=arguments-differ
+        try:
+            self.metadata(path)
+            return True
+        except FileNotFoundError:
+            return False
+
+    def isdir(self, path):  # pylint: disable=arguments-differ
+        try:
+            meta = self.metadata(path)
+            return meta.isdir
+        except FileNotFoundError:
+            return False
+
+    def check_isdir(self, path_info, outs):
+        if len(outs) != 1:
+            return True
+
+        out = outs[0]
+        if not out.is_dir_checksum:
+            return out.path_info != path_info
+        if out.path_info == path_info:
+            return True
+
+        try:
+            self._get_granular_hash(path_info, out)
+            return False
+        except FileNotFoundError:
+            return True
+
+    def isfile(self, path):  # pylint: disable=arguments-differ
+        try:
+            meta = self.metadata(path)
+            return meta.isfile
+        except FileNotFoundError:
+            return False
+
+    def _fetch_dir(self, out, **kwargs):
+        # pull dir cache if needed
+        out.get_dir_cache(**kwargs)
+
+        if not out.obj:
+            raise FileNotFoundError
+
+    def _add_dir(self, trie, out, **kwargs):
+        self._fetch_dir(out, **kwargs)
+
+        base = out.path_info.parts
+        for key, _ in out.obj:  # noqa: B301
+            trie[base + key] = None
+
+    def _walk(self, root, trie, topdown=True, **kwargs):
+        dirs = set()
+        files = []
+
+        out = trie.get(root.parts)
+        if out and out.is_dir_checksum:
+            self._add_dir(trie, out, **kwargs)
+
+        root_len = len(root.parts)
+        for key, out in trie.iteritems(prefix=root.parts):  # noqa: B301
+            if key == root.parts:
+                continue
+
+            name = key[root_len]
+            if len(key) > root_len + 1 or (out and out.is_dir_checksum):
+                dirs.add(name)
+                continue
+
+            files.append(name)
+
+        assert topdown
+        dirs = list(dirs)
+        yield root.fspath, dirs, files
+
+        for dname in dirs:
+            yield from self._walk(root / dname, trie)
+
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        from pygtrie import Trie
+
+        assert topdown
+        root = PathInfo(os.path.abspath(top))
+        try:
+            meta = self.metadata(root)
+        except FileNotFoundError:
+            if onerror is not None:
+                onerror(FileNotFoundError(top))
+            return
+
+        if not meta.isdir:
+            if onerror is not None:
+                onerror(NotADirectoryError(top))
+            return
+
+        trie = Trie()
+        for out in meta.outs:
+            trie[out.path_info.parts] = out
+
+            if out.is_dir_checksum and root.isin_or_eq(out.path_info):
+                self._add_dir(trie, out, **kwargs)
+
+        yield from self._walk(root, trie, topdown=topdown, **kwargs)
+
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info):
+            for fname in files:
+                # NOTE: os.path.join is ~5.5 times slower
+                yield PathInfo(f"{root}{os.sep}{fname}")
+
+    def isdvc(self, path, recursive=False, strict=True):
+        try:
+            meta = self.metadata(path)
+        except FileNotFoundError:
+            return False
+
+        recurse = recursive or not strict
+        return meta.output_exists if recurse else meta.is_output
+
+    def isexec(self, path_info):  # pylint: disable=unused-argument
+        return False
+
+    def metadata(self, path_info):
+        path_info = PathInfo(os.path.abspath(path_info))
+
+        try:
+            outs = self._find_outs(path_info, strict=False, recursive=True)
+        except OutputNotFoundError as exc:
+            raise FileNotFoundError from exc
+
+        meta = Metadata(path_info=path_info, outs=outs, repo=self.repo)
+        meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs)
+        return meta
+
+    def info(self, path_info):
+        meta = self.metadata(path_info)
+        ret = {"type": "dir" if meta.isdir else "file"}
+        if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
+            hash_info = meta.outs[0].hash_info
+            ret["size"] = hash_info.size
+            ret[hash_info.name] = hash_info.value
+        elif meta.part_of_output:
+            (out,) = meta.outs
+            key = path_info.relative_to(out.path_info).parts
+            obj = out.obj.trie.get(key)
+            if obj:
+                ret["size"] = obj.size
+                ret[obj.hash_info.name] = obj.hash_info.value
+
+        return ret

dvc/fs/fsspec_wrapper.py

@@ -0,0 +1,158 @@
+import os
+import shutil
+from functools import lru_cache
+
+from dvc.progress import Tqdm
+
+from .base import BaseFileSystem
+
+
+# pylint: disable=no-member
+class FSSpecWrapper(BaseFileSystem):
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+        self.fs_args = {"skip_instance_cache": True}
+        self.fs_args.update(self._prepare_credentials(config))
+
+    def fs(self):
+        raise NotImplementedError
+
+    @lru_cache(512)
+    def _with_bucket(self, path):
+        if isinstance(path, self.PATH_CLS):
+            return f"{path.bucket}/{path.path}"
+        return path
+
+    def _strip_bucket(self, entry):
+        try:
+            bucket, path = entry.split("/", 1)
+        except ValueError:
+            # If there is no path attached, only returns
+            # the bucket (top-level).
+            bucket, path = entry, None
+
+        return path or bucket
+
+    def _strip_buckets(self, entries, detail, prefix=None):
+        for entry in entries:
+            if detail:
+                entry = self._entry_hook(entry.copy())
+                entry["name"] = self._strip_bucket(entry["name"])
+            else:
+                entry = self._strip_bucket(
+                    f"{prefix}/{entry}" if prefix else entry
+                )
+            yield entry
+
+    def _entry_hook(self, entry):
+        """Simple hook method to be overriden when wanted to process
+        entries within info() and ls(detail=True) calls"""
+        return entry
+
+    def _prepare_credentials(self, config):  # pylint: disable=unused-argument
+        """Prepare the arguments for authentication to the
+        host filesystem"""
+        return {}
+
+    def isdir(self, path_info):
+        if not self.exists(path_info):
+            return False
+
+        # Directory in object storages are interpreted differently
+        # among different fsspec providers, so this logic is a temporary
+        # measure for us to adapt as of now. It checks whether it is a
+        # directory (as in a prefix with contents) or whether it is an empty
+        # file where it's name ends with a forward slash
+        entry = self.info(path_info)
+        return entry["type"] == "directory" or (
+            entry["size"] == 0
+            and entry["type"] == "file"
+            and entry["name"].endswith("/")
+        )
+
+    def isfile(self, path_info):
+        if not self.exists(path_info):
+            return False
+
+        return not self.isdir(path_info)
+
+    def is_empty(self, path_info):
+        entry = self.info(path_info)
+        if entry["type"] == "directory":
+            return not self.fs.ls(self._with_bucket(path_info))
+        return entry["size"] == 0
+
+    def open(
+        self, path_info, mode="r", **kwargs
+    ):  # pylint: disable=arguments-differ
+        return self.fs.open(self._with_bucket(path_info), mode=mode)
+
+    def copy(self, from_info, to_info):
+        self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
+
+    def exists(self, path_info, use_dvcignore=False):
+        return self.fs.exists(self._with_bucket(path_info))
+
+    def ls(
+        self, path_info, detail=False, recursive=False
+    ):  # pylint: disable=arguments-differ
+        if self.isdir(path_info) and self.is_empty(path_info):
+            return None
+
+        path = self._with_bucket(path_info)
+        if recursive:
+            for root, _, files in self.fs.walk(path, detail=detail):
+                if detail:
+                    files = files.values()
+                yield from self._strip_buckets(files, detail, prefix=root)
+            return None
+
+        yield from self._strip_buckets(self.ls(path, detail=detail), detail)
+
+    def walk_files(self, path_info, **kwargs):
+        for file in self.ls(path_info, recursive=True):
+            yield path_info.replace(path=file)
+
+    def remove(self, path_info):
+        self.fs.rm(self._with_bucket(path_info))
+
+    def info(self, path_info):
+        info = self.fs.info(self._with_bucket(path_info)).copy()
+        info = self._entry_hook(info)
+        info["name"] = self._strip_bucket(info["name"])
+        return info
+
+    def _upload_fobj(self, fobj, to_info):
+        with self.open(to_info, "wb") as fdest:
+            shutil.copyfileobj(fobj, fdest, length=fdest.blocksize)
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **kwargs
+    ):
+        total = os.path.getsize(from_file)
+        with open(from_file, "rb") as fobj:
+            self.upload_fobj(
+                fobj,
+                self._with_bucket(to_info),
+                desc=name,
+                total=total,
+                no_progress_bar=no_progress_bar,
+            )
+        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
+
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
+    ):
+        total = self.getsize(self._with_bucket(from_info))
+        with self.open(from_info, "rb") as fobj:
+            with Tqdm.wrapattr(
+                fobj,
+                "read",
+                desc=name,
+                disable=no_progress_bar,
+                bytes=True,
+                total=total,
+                **pbar_args,
+            ) as wrapped:
+                with open(to_file, "wb") as fdest:
+                    shutil.copyfileobj(wrapped, fdest, length=fobj.blocksize)

dvc/fs/gdrive.py

@@ -18,7 +18,7 @@ from dvc.scheme import Schemes
 from dvc.utils import format_link, tmp_fname
 from dvc.utils.stream import IterStream
 
-from .base import BaseTree
+from .base import BaseFileSystem
 
 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
@@ -85,11 +85,11 @@ class GDriveURLInfo(CloudURLInfo):
         self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
 
 
-class GDriveTree(BaseTree):
+class GDriveFileSystem(BaseFileSystem):
     scheme = Schemes.GDRIVE
     PATH_CLS = GDriveURLInfo
+    PARAM_CHECKSUM = "checksum"
     REQUIRES = {"pydrive2": "pydrive2"}
-    DEFAULT_VERIFY = True
     # Always prefer traverse for GDrive since API usage quotas are a concern.
     TRAVERSE_WEIGHT_MULTIPLIER = 1
     TRAVERSE_PREFIX_LEN = 2
@@ -114,23 +114,24 @@ class GDriveTree(BaseTree):
             )
 
         self._bucket = self.path_info.bucket
+        self._path = self.path_info.path
         self._trash_only = config.get("gdrive_trash_only")
         self._use_service_account = config.get("gdrive_use_service_account")
-        self._service_account_email = config.get(
-            "gdrive_service_account_email"
-        )
         self._service_account_user_email = config.get(
             "gdrive_service_account_user_email"
         )
-        self._service_account_p12_file_path = config.get(
-            "gdrive_service_account_p12_file_path"
+        self._service_account_json_file_path = config.get(
+            "gdrive_service_account_json_file_path"
         )
         self._client_id = config.get("gdrive_client_id")
         self._client_secret = config.get("gdrive_client_secret")
         self._validate_config()
+        self._gdrive_service_credentials_path = tmp_fname(
+            os.path.join(self.repo.tmp_dir, "")
+        )
         self._gdrive_user_credentials_path = (
             tmp_fname(os.path.join(self.repo.tmp_dir, ""))
-            if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA)
+            if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
             else config.get(
                 "gdrive_user_credentials_file",
                 os.path.join(
@@ -141,14 +142,14 @@ class GDriveTree(BaseTree):
 
     def _validate_config(self):
         # Validate Service Account configuration
-        if self._use_service_account and (
-            not self._service_account_email
-            or not self._service_account_p12_file_path
+        if (
+            self._use_service_account
+            and not self._service_account_json_file_path
         ):
             raise DvcException(
-                "To use service account, set `gdrive_service_account_email`,\n"
-                "`gdrive_service_account_p12_file_path`, and optionally "
-                "`gdrive_service_account_user_email`\nin DVC config. "
+                "To use service account, set "
+                "`gdrive_service_account_json_file_path`, and optionally"
+                "`gdrive_service_account_user_email` in DVC config\n"
                 "Learn more at {}".format(
                     format_link("https://man.dvc.org/remote/modify")
                 )
@@ -172,8 +173,8 @@ class GDriveTree(BaseTree):
         Useful for tests, exception messages, etc. Returns either env variable
         name if it's set or actual path to the credentials file.
         """
-        if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
-            return GDriveTree.GDRIVE_CREDENTIALS_DATA
+        if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
+            return GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
         if os.path.exists(self._gdrive_user_credentials_path):
             return self._gdrive_user_credentials_path
         return None
@@ -187,7 +188,7 @@ class GDriveTree(BaseTree):
         DVC config client id or secret but forgets to remove the cached
         credentials file.
         """
-        if not os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
+        if not os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
             if (
                 settings["client_config"]["client_id"]
                 != auth.credentials.client_id
@@ -210,9 +211,18 @@ class GDriveTree(BaseTree):
         from pydrive2.auth import GoogleAuth
         from pydrive2.drive import GoogleDrive
 
-        if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
-            with open(self._gdrive_user_credentials_path, "w") as cred_file:
-                cred_file.write(os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA))
+        temporary_save_path = self._gdrive_user_credentials_path
+        is_credentials_temp = os.getenv(
+            GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
+        )
+        if self._use_service_account:
+            temporary_save_path = self._gdrive_service_credentials_path
+
+        if is_credentials_temp:
+            with open(temporary_save_path, "w") as cred_file:
+                cred_file.write(
+                    os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
+                )
 
         auth_settings = {
             "client_config_backend": "settings",
@@ -228,10 +238,14 @@ class GDriveTree(BaseTree):
 
         if self._use_service_account:
             auth_settings["service_config"] = {
-                "client_service_email": self._service_account_email,
                 "client_user_email": self._service_account_user_email,
-                "client_pkcs12_file_path": self._service_account_p12_file_path,
+                "client_json_file_path": self._service_account_json_file_path,
             }
+            if is_credentials_temp:
+                auth_settings["service_config"][
+                    "client_json_file_path"
+                ] = temporary_save_path
+
         else:
             auth_settings["client_config"] = {
                 "client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
@@ -258,7 +272,7 @@ class GDriveTree(BaseTree):
                 gauth.ServiceAuth()
             else:
                 gauth.CommandLineAuth()
-                GDriveTree._validate_credentials(gauth, auth_settings)
+                GDriveFileSystem._validate_credentials(gauth, auth_settings)
 
         # Handle AuthenticationError, RefreshError and other auth failures
         # It's hard to come up with a narrow exception, since PyDrive throws
@@ -267,8 +281,8 @@ class GDriveTree(BaseTree):
         except Exception as exc:
             raise GDriveAuthError(self.credentials_location) from exc
         finally:
-            if os.getenv(GDriveTree.GDRIVE_CREDENTIALS_DATA):
-                os.remove(self._gdrive_user_credentials_path)
+            if is_credentials_temp:
+                os.remove(temporary_save_path)
 
         return GoogleDrive(gauth)
 
@@ -285,20 +299,21 @@ class GDriveTree(BaseTree):
             ),
         }
 
-        self._cache_path_id(self.path_info.path, cache["root_id"], cache)
+        self._cache_path_id(self._path, cache["root_id"], cache=cache)
 
         for item in self._gdrive_list(
             "'{}' in parents and trashed=false".format(cache["root_id"])
         ):
             item_path = (self.path_info / item["title"]).path
-            self._cache_path_id(item_path, item["id"], cache)
+            self._cache_path_id(item_path, item["id"], cache=cache)
 
         return cache
 
-    def _cache_path_id(self, path, item_id, cache=None):
+    def _cache_path_id(self, path, *item_ids, cache=None):
         cache = cache or self._ids_cache
-        cache["dirs"][path].append(item_id)
-        cache["ids"][item_id] = path
+        for item_id in item_ids:
+            cache["dirs"][path].append(item_id)
+            cache["ids"][item_id] = path
 
     @cached_property
     def _list_params(self):
@@ -345,32 +360,12 @@ class GDriveTree(BaseTree):
         return item.get("driveId", None)
 
     @_gdrive_retry
-    def _gdrive_upload_file(
-        self,
-        parent_id,
-        title,
-        no_progress_bar=False,
-        from_file="",
-        progress_name="",
-    ):
+    def _gdrive_upload_fobj(self, fobj, parent_id, title):
         item = self._drive.CreateFile(
             {"title": title, "parents": [{"id": parent_id}]}
         )
-
-        with open(from_file, "rb") as fobj:
-            total = os.path.getsize(from_file)
-            with Tqdm.wrapattr(
-                fobj,
-                "read",
-                desc=progress_name,
-                total=total,
-                disable=no_progress_bar,
-            ) as wrapped:
-                # PyDrive doesn't like content property setting for empty files
-                # https://github.com/gsuitedevs/PyDrive/issues/121
-                if total:
-                    item.content = wrapped
-                item.Upload()
+        item.content = fobj
+        item.Upload()
         return item
 
     @_gdrive_retry
@@ -392,7 +387,7 @@ class GDriveTree(BaseTree):
 
     @contextmanager
     @_gdrive_retry
-    def open(self, path_info, mode="r", encoding=None):
+    def open(self, path_info, mode="r", encoding=None, **kwargs):
         assert mode in {"r", "rt", "rb"}
 
         item_id = self._get_item_id(path_info)
@@ -497,7 +492,7 @@ class GDriveTree(BaseTree):
             return self._ids_cache["dirs"].get(path, [])
         return []
 
-    def _path_to_item_ids(self, path, create, use_cache):
+    def _path_to_item_ids(self, path, create=False, use_cache=True):
         item_ids = self._get_cached_item_ids(path, use_cache)
         if item_ids:
             return item_ids
@@ -530,51 +525,116 @@ class GDriveTree(BaseTree):
         else:
             return True
 
-    def _list_paths(self, prefix=None):
-        if not self._ids_cache["ids"]:
-            return
-
-        if prefix:
-            dir_ids = self._ids_cache["dirs"].get(prefix[:2])
-            if not dir_ids:
-                return
-        else:
-            dir_ids = self._ids_cache["ids"]
-        parents_query = " or ".join(
-            f"'{dir_id}' in parents" for dir_id in dir_ids
+    def _gdrive_list_ids(self, query_ids):
+        query = " or ".join(
+            f"'{query_id}' in parents" for query_id in query_ids
         )
-        query = f"({parents_query}) and trashed=false"
+        query = f"({query}) and trashed=false"
+        return self._gdrive_list(query)
 
-        for item in self._gdrive_list(query):
-            parent_id = item["parents"][0]["id"]
-            yield posixpath.join(
-                self._ids_cache["ids"][parent_id], item["title"]
-            )
+    def _ls_recursive(self, path_info, detail=False):
+        root_path = path_info.path
+        seen_paths = set()
+
+        dir_ids = [self._ids_cache["ids"].copy()]
+        while dir_ids:
+            query_ids = {
+                dir_id: dir_name
+                for dir_id, dir_name in dir_ids.pop().items()
+                if posixpath.commonpath([root_path, dir_name]) == root_path
+                if dir_id not in seen_paths
+            }
+            if not query_ids:
+                continue
+
+            seen_paths |= query_ids.keys()
+
+            new_query_ids = {}
+            dir_ids.append(new_query_ids)
+            for item in self._gdrive_list_ids(query_ids):
+                parent_id = item["parents"][0]["id"]
+                item_path = posixpath.join(query_ids[parent_id], item["title"])
+                if item["mimeType"] == FOLDER_MIME_TYPE:
+                    new_query_ids[item["id"]] = item_path
+                    self._cache_path_id(item_path, item["id"])
+                    continue
+
+                if detail:
+                    yield {
+                        "type": "file",
+                        "name": item_path,
+                        "size": item["fileSize"],
+                        "checksum": item["md5Checksum"],
+                    }
+                else:
+                    yield item_path
+
+    def ls(
+        self, path_info, detail=False, recursive=False
+    ):  # pylint: disable=arguments-differ
+        if recursive:
+            yield from self._ls_recursive(path_info, detail=detail)
+            return None
+
+        cached = path_info.path in self._ids_cache["dirs"]
+        if cached:
+            dir_ids = self._ids_cache["dirs"][path_info.path]
+        else:
+            dir_ids = self._path_to_item_ids(path_info.path)
+
+        if not dir_ids:
+            return None
+
+        root_path = path_info.path
+        for item in self._gdrive_list_ids(dir_ids):
+            item_path = posixpath.join(root_path, item["title"])
+            if detail:
+                if item["mimeType"] == FOLDER_MIME_TYPE:
+                    yield {"type": "directory", "name": item_path}
+                else:
+                    yield {
+                        "type": "file",
+                        "name": item_path,
+                        "size": item["fileSize"],
+                        "checksum": item["md5Checksum"],
+                    }
+            else:
+                yield item_path
+
+        if not cached:
+            self._cache_path_id(root_path, *dir_ids)
 
     def walk_files(self, path_info, **kwargs):
-        use_prefix = kwargs.pop("prefix", False)
-        if path_info == self.path_info or not use_prefix:
-            prefix = None
-        else:
-            prefix = path_info.path
-        for fname in self._list_paths(prefix=prefix, **kwargs):
-            yield path_info.replace(fname)
+        for filename in self.ls(path_info, recursive=True):
+            yield path_info.replace(path=filename)
 
     def remove(self, path_info):
         item_id = self._get_item_id(path_info)
         self.gdrive_delete_file(item_id)
 
-    def get_file_hash(self, path_info):
-        raise NotImplementedError
+    def info(self, path_info):
+        item_id = self._get_item_id(path_info)
+        gdrive_file = self._drive.CreateFile({"id": item_id})
+        gdrive_file.FetchMetadata(fields="fileSize")
+        return {"size": gdrive_file.get("fileSize")}
 
-    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
+    def _upload_fobj(self, fobj, to_info):
         dirname = to_info.parent
         assert dirname
-        parent_id = self._get_item_id(dirname, True)
+        parent_id = self._get_item_id(dirname, create=True)
+        self._gdrive_upload_fobj(fobj, parent_id, to_info.name)
 
-        self._gdrive_upload_file(
-            parent_id, to_info.name, no_progress_bar, from_file, name
-        )
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+    ):
+        with open(from_file, "rb") as fobj:
+            self.upload_fobj(
+                fobj,
+                to_info,
+                no_progress_bar=no_progress_bar,
+                desc=name or to_info.name,
+                total=os.path.getsize(from_file),
+            )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
         item_id = self._get_item_id(from_info)

dvc/fs/git.py

@@ -0,0 +1,168 @@
+import errno
+import os
+
+from funcy import cached_property
+
+from dvc.utils import is_exec, relpath
+
+from .base import BaseFileSystem
+
+
+class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
+    """Proxies the repo file access methods to Git objects"""
+
+    def __init__(
+        self, root_dir, trie, use_dvcignore=False, dvcignore_root=None
+    ):
+        super().__init__(None, {})
+        self._fs_root = root_dir
+        self.trie = trie
+        self.use_dvcignore = use_dvcignore
+        self.dvcignore_root = dvcignore_root
+
+    @property
+    def rev(self):
+        return self.trie.rev
+
+    @property
+    def fs_root(self):
+        return self._fs_root
+
+    def _get_key(self, path):
+        if isinstance(path, str):
+            if not os.path.isabs(path):
+                relparts = path.split(os.sep)
+            else:
+                relparts = relpath(path, self.fs_root).split(os.sep)
+        else:
+            relparts = path.relative_to(self.fs_root).parts
+        if relparts == ["."]:
+            return ()
+        return tuple(relparts)
+
+    @cached_property
+    def dvcignore(self):
+        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
+
+        root = self.dvcignore_root or self.fs_root
+        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
+        return cls(self, root)
+
+    def open(
+        self, path, mode="r", encoding=None
+    ):  # pylint: disable=arguments-differ
+        # NOTE: this is incorrect, we need to use locale to determine default
+        # encoding.
+        encoding = encoding or "utf-8"
+
+        key = self._get_key(path)
+        try:
+            return self.trie.open(key, mode=mode, encoding=encoding)
+        except KeyError as exc:
+            msg = os.strerror(errno.ENOENT) + f"in branch '{self.rev}'"
+            raise FileNotFoundError(errno.ENOENT, msg, path) from exc
+        except IsADirectoryError as exc:
+            raise IsADirectoryError(
+                errno.EISDIR, os.strerror(errno.EISDIR), path
+            ) from exc
+
+    def exists(
+        self, path, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
+        def _is_ignored(path):
+            return self.dvcignore.is_ignored_file(
+                path
+            ) or self.dvcignore.is_ignored_dir(path)
+
+        if use_dvcignore and _is_ignored(path):
+            return False
+
+        key = self._get_key(path)
+        return self.trie.exists(key)
+
+    def isdir(
+        self, path, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
+        if use_dvcignore and self.dvcignore.is_ignored_dir(path):
+            return False
+        key = self._get_key(path)
+        return self.trie.isdir(key)
+
+    def isfile(self, path):  # pylint: disable=arguments-differ
+        if self.dvcignore.is_ignored_file(path):
+            return False
+        key = self._get_key(path)
+        return self.trie.isfile(key)
+
+    def walk(
+        self,
+        top,
+        topdown=True,
+        onerror=None,
+        use_dvcignore=True,
+        ignore_subrepos=True,
+    ):
+        """Directory tree generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        """
+        if not self.isdir(top, use_dvcignore=use_dvcignore):
+            if onerror:
+                if self.exists(top, use_dvcignore=use_dvcignore):
+                    exc = NotADirectoryError(
+                        errno.ENOTDIR, os.strerror(errno.ENOTDIR), top
+                    )
+                else:
+                    exc = FileNotFoundError(
+                        errno.ENOENT, os.strerror(errno.ENOENT), top
+                    )
+                onerror(exc)
+            return []
+
+        key = self._get_key(top)
+        for prefix, dirs, files in self.trie.walk(key, topdown=topdown):
+            if prefix:
+                root = os.path.join(self.fs_root, os.sep.join(prefix))
+            else:
+                root = self.fs_root
+            if use_dvcignore:
+                dirs[:], files[:] = self.dvcignore(
+                    root, dirs, files, ignore_subrepos=ignore_subrepos,
+                )
+            yield root, dirs, files
+
+    def isexec(self, path_info):
+        if not self.exists(path_info):
+            return False
+
+        mode = self.stat(path_info).st_mode
+        return is_exec(mode)
+
+    def info(self, path_info):
+        key = self._get_key(path_info)
+        try:
+            st = self.trie.stat(key)
+        except KeyError:
+            raise FileNotFoundError(
+                errno.ENOENT, os.strerror(errno.ENOENT), path_info
+            )
+        return {"size": st.st_size}
+
+    def stat(self, path):
+        key = self._get_key(path)
+        try:
+            return self.trie.stat(key)
+        except KeyError:
+            raise FileNotFoundError(
+                errno.ENOENT, os.strerror(errno.ENOENT), path
+            )
+
+    def walk_files(self, top):  # pylint: disable=arguments-differ
+        for root, _, files in self.walk(top):
+            for file in files:
+                # NOTE: os.path.join is ~5.5 times slower
+                yield f"{root}{os.sep}{file}"
+
+    def _reset(self):
+        return self.__dict__.pop("dvcignore", None)

dvc/fs/gs.py

@@ -0,0 +1,41 @@
+import base64
+import threading
+
+from funcy import cached_property, wrap_prop
+
+from dvc.path_info import CloudURLInfo
+from dvc.scheme import Schemes
+
+from .fsspec_wrapper import FSSpecWrapper
+
+
+class GSFileSystem(FSSpecWrapper):
+    scheme = Schemes.GS
+    PATH_CLS = CloudURLInfo
+    REQUIRES = {"gcsfs": "gcsfs"}
+    PARAM_CHECKSUM = "etag"
+    DETAIL_FIELDS = frozenset(("etag", "size"))
+
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+
+        url = config.get("url", "gs://")
+        self.path_info = self.PATH_CLS(url)
+
+    def _prepare_credentials(self, config):
+        login_info = {"consistency": None}
+        login_info["project"] = config.get("projectname")
+        login_info["token"] = config.get("credentialpath")
+        return login_info
+
+    def _entry_hook(self, entry):
+        if "etag" in entry:
+            entry["etag"] = base64.b64decode(entry["etag"]).hex()
+        return entry
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def fs(self):
+        from gcsfs import GCSFileSystem
+
+        return GCSFileSystem(**self.fs_args)

dvc/fs/hdfs.py

@@ -0,0 +1,275 @@
+import io
+import logging
+import os
+import re
+import shutil
+import subprocess
+from collections import deque
+from contextlib import closing, contextmanager
+from urllib.parse import urlparse
+
+from dvc.hash_info import HashInfo
+from dvc.progress import Tqdm
+from dvc.scheme import Schemes
+from dvc.utils import fix_env, tmp_fname
+
+from .base import BaseFileSystem, RemoteCmdError
+from .pool import get_connection
+
+logger = logging.getLogger(__name__)
+
+
+def _hadoop_fs(cmd, user=None):
+    cmd = "hadoop fs -" + cmd
+    if user:
+        cmd = f"HADOOP_USER_NAME={user} " + cmd
+
+    # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
+    # See https://github.com/iterative/dvc/issues/1197.
+    close_fds = os.name != "nt"
+
+    executable = os.getenv("SHELL") if os.name != "nt" else None
+    p = subprocess.Popen(
+        cmd,
+        shell=True,
+        close_fds=close_fds,
+        executable=executable,
+        env=fix_env(os.environ),
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    out, err = p.communicate()
+    if p.returncode != 0:
+        raise RemoteCmdError("hdfs", cmd, p.returncode, err)
+    return out.decode("utf-8")
+
+
+def _group(regex, s, gname):
+    match = re.match(regex, s)
+    assert match is not None
+    return match.group(gname)
+
+
+def _hadoop_fs_checksum(path_info):
+    # NOTE: pyarrow doesn't support checksum, so we need to use hadoop
+    regex = r".*\t.*\t(?P<checksum>.*)"
+    stdout = _hadoop_fs(f"checksum {path_info.url}", user=path_info.user)
+    return _group(regex, stdout, "checksum")
+
+
+class HDFSFileSystem(BaseFileSystem):
+    scheme = Schemes.HDFS
+    REQUIRES = {"pyarrow": "pyarrow"}
+    REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
+    PARAM_CHECKSUM = "checksum"
+    TRAVERSE_PREFIX_LEN = 2
+
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+
+        self.path_info = None
+        url = config.get("url")
+        if not url:
+            return
+
+        parsed = urlparse(url)
+        user = parsed.username or config.get("user")
+
+        self.path_info = self.PATH_CLS.from_parts(
+            scheme=self.scheme,
+            host=parsed.hostname,
+            user=user,
+            port=parsed.port,
+            path=parsed.path,
+        )
+
+    @staticmethod
+    def hdfs(path_info):
+        import pyarrow.fs
+
+        # NOTE: HadoopFileSystem is not meant to be closed by us and doesn't
+        # have close() method or alternative, so we add a noop one to satisfy
+        # our connection pool.
+        class HDFSConnection(pyarrow.fs.HadoopFileSystem):
+            def close(self):
+                pass
+
+        return get_connection(
+            HDFSConnection,
+            path_info.host,
+            path_info.port,
+            user=path_info.user,
+        )
+
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None, **kwargs):
+        assert mode in {"r", "rt", "rb"}
+
+        try:
+            with self.hdfs(path_info) as hdfs, closing(
+                hdfs.open_input_stream(path_info.path)
+            ) as fd:
+                if mode == "rb":
+                    yield fd
+                else:
+                    yield io.TextIOWrapper(fd, encoding=encoding)
+        except OSError as e:
+            # Empty .errno and not specific enough error class in pyarrow,
+            # see https://issues.apache.org/jira/browse/ARROW-6248
+            if "file does not exist" in str(e):
+                raise FileNotFoundError(*e.args)
+            raise
+
+    def exists(self, path_info, use_dvcignore=True):
+        assert not isinstance(path_info, list)
+        assert path_info.scheme == "hdfs"
+        with self.hdfs(path_info) as hdfs:
+            import pyarrow.fs
+
+            file_info = hdfs.get_file_info(path_info.path)
+            return file_info.type != pyarrow.fs.FileType.NotFound
+
+    def _walk(self, hdfs, root, topdown=True):
+        import posixpath
+
+        from pyarrow.fs import FileSelector, FileType
+
+        dirs = deque()
+        nondirs = deque()
+
+        selector = FileSelector(root)
+        for entry in hdfs.get_file_info(selector):
+            if entry.type == FileType.Directory:
+                dirs.append(entry.base_name)
+            else:
+                nondirs.append(entry.base_name)
+
+        if topdown:
+            yield root, dirs, nondirs
+
+        for dname in dirs:
+            # NOTE: posixpath.join() is slower
+            yield from self._walk(
+                hdfs, f"{root}{posixpath.sep}{dname}", topdown=topdown
+            )
+
+        if not topdown:
+            yield root, dirs, nondirs
+
+    def walk(self, path_info, **kwargs):
+        if not self.isdir(path_info):
+            return
+
+        with self.hdfs(path_info) as hdfs:
+            for root, dnames, fnames in self._walk(
+                hdfs, path_info.path, **kwargs
+            ):
+                yield path_info.replace(path=root), dnames, fnames
+
+    def walk_files(self, path_info, **kwargs):
+        for root, _, fnames in self.walk(path_info):
+            yield from (root / fname for fname in fnames)
+
+    def remove(self, path_info):
+        if path_info.scheme != "hdfs":
+            raise NotImplementedError
+
+        if self.exists(path_info):
+            logger.debug(f"Removing {path_info.path}")
+            with self.hdfs(path_info) as hdfs:
+                import pyarrow.fs
+
+                file_info = hdfs.get_file_info(path_info.path)
+                if file_info.type == pyarrow.fs.FileType.Directory:
+                    hdfs.delete_dir(path_info.path)
+                else:
+                    hdfs.delete_file(path_info.path)
+
+    def makedirs(self, path_info):
+        with self.hdfs(path_info) as hdfs:
+            # NOTE: fs.create_dir creates parents by default
+            hdfs.create_dir(path_info.path)
+
+    def copy(self, from_info, to_info, **_kwargs):
+        # NOTE: hdfs.copy_file is not supported yet in pyarrow
+        with self.hdfs(to_info) as hdfs:
+            # NOTE: this is how `hadoop fs -cp` works too: it copies through
+            # your local machine.
+            with closing(hdfs.open_input_stream(from_info.path)) as from_fobj:
+                tmp_info = to_info.parent / tmp_fname(to_info.name)
+                try:
+                    with closing(
+                        hdfs.open_output_stream(tmp_info.path)
+                    ) as tmp_fobj:
+                        shutil.copyfileobj(from_fobj, tmp_fobj)
+                    hdfs.move(tmp_info.path, to_info.path)
+                except Exception:
+                    self.remove(tmp_info)
+                    raise
+
+    def isfile(self, path_info):
+        with self.hdfs(path_info) as hdfs:
+            import pyarrow.fs
+
+            file_info = hdfs.get_file_info(path_info.path)
+            return file_info.type == pyarrow.fs.FileType.File
+
+    def isdir(self, path_info):
+        with self.hdfs(path_info) as hdfs:
+            import pyarrow.fs
+
+            file_info = hdfs.get_file_info(path_info.path)
+            return file_info.type == pyarrow.fs.FileType.Directory
+
+    def info(self, path_info):
+        with self.hdfs(path_info) as hdfs:
+            finfo = hdfs.get_file_info(path_info.path)
+            return {"size": finfo.size}
+
+    def checksum(self, path_info):
+        return HashInfo(
+            "checksum",
+            _hadoop_fs_checksum(path_info),
+            size=self.getsize(path_info),
+        )
+
+    def _upload_fobj(self, fobj, to_info):
+        with self.hdfs(to_info) as hdfs:
+            with hdfs.open_output_stream(to_info.path) as fdest:
+                shutil.copyfileobj(fobj, fdest)
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+    ):
+        with self.hdfs(to_info) as hdfs:
+            tmp_file = tmp_fname(to_info.path)
+            total = os.path.getsize(from_file)
+            with open(from_file, "rb") as fobj:
+                with Tqdm.wrapattr(
+                    fobj,
+                    "read",
+                    desc=name,
+                    total=total,
+                    disable=no_progress_bar,
+                ) as wrapped:
+                    with hdfs.open_output_stream(tmp_file) as sobj:
+                        sobj.write(wrapped.read())
+            hdfs.move(tmp_file, to_info.path)
+
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        with self.hdfs(from_info) as hdfs:
+            file_info = hdfs.get_file_info(from_info.path)
+            total = file_info.size
+            with open(to_file, "wb+") as fobj:
+                with Tqdm.wrapattr(
+                    fobj,
+                    "write",
+                    desc=name,
+                    total=total,
+                    disable=no_progress_bar,
+                ) as wrapped:
+                    with hdfs.open_input_stream(from_info.path) as sobj:
+                        wrapped.write(sobj.read())

dvc/fs/http.py

@@ -1,6 +1,7 @@
 import logging
 import os.path
 import threading
+from typing import Optional
 
 from funcy import cached_property, memoize, wrap_prop, wrap_with
 
@@ -10,7 +11,7 @@ from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
 
-from .base import BaseTree
+from .base import BaseFileSystem
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +25,7 @@ def ask_password(host, user):
     )
 
 
-class HTTPTree(BaseTree):  # pylint:disable=abstract-method
+class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     scheme = Schemes.HTTP
     PATH_CLS = HTTPURLInfo
     PARAM_CHECKSUM = "etag"
@@ -32,7 +33,7 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
 
     SESSION_RETRIES = 5
     SESSION_BACKOFF_FACTOR = 0.1
-    REQUEST_TIMEOUT = 10
+    REQUEST_TIMEOUT = 60
     CHUNK_SIZE = 2 ** 16
 
     def __init__(self, repo, config):
@@ -41,9 +42,8 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
         url = config.get("url")
         if url:
             self.path_info = self.PATH_CLS(url)
-            user = config.get("user", None)
-            if user:
-                self.path_info.user = user
+            self.user = config.get("user", None)
+            self.host = self.path_info.host
         else:
             self.path_info = None
 
@@ -52,25 +52,26 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
         self.password = config.get("password", None)
         self.ask_password = config.get("ask_password", False)
         self.headers = {}
+        self.ssl_verify = config.get("ssl_verify", True)
+        self.method = config.get("method", "POST")
 
-    def _auth_method(self, path_info=None):
+    def _auth_method(self):
         from requests.auth import HTTPBasicAuth, HTTPDigestAuth
 
-        if path_info is None:
-            path_info = self.path_info
-
         if self.auth:
             if self.ask_password and self.password is None:
-                host, user = path_info.host, path_info.user
-                self.password = ask_password(host, user)
+                self.password = ask_password(self.host, self.user)
             if self.auth == "basic":
-                return HTTPBasicAuth(path_info.user, self.password)
+                return HTTPBasicAuth(self.user, self.password)
             if self.auth == "digest":
-                return HTTPDigestAuth(path_info.user, self.password)
+                return HTTPDigestAuth(self.user, self.password)
             if self.auth == "custom" and self.custom_auth_header:
                 self.headers.update({self.custom_auth_header: self.password})
         return None
 
+    def _generate_download_url(self, path_info):
+        return path_info.url
+
     @wrap_prop(threading.Lock())
     @cached_property
     def _session(self):
@@ -80,6 +81,8 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
 
         session = requests.Session()
 
+        session.verify = self.ssl_verify
+
         retries = Retry(
             total=self.SESSION_RETRIES,
             backoff_factor=self.SESSION_BACKOFF_FACTOR,
@@ -122,27 +125,44 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
         except requests.exceptions.RequestException:
             raise DvcException(f"could not perform a {method} request")
 
+    def _head(self, url):
+        response = self.request("HEAD", url)
+        if response.ok:
+            return response
+
+        # Sometimes servers are configured to forbid HEAD requests
+        # Context: https://github.com/iterative/dvc/issues/4131
+        with self.request("GET", url, stream=True) as r:
+            if r.ok:
+                return r
+
+        return response
+
     def exists(self, path_info, use_dvcignore=True):
-        return bool(self.request("HEAD", path_info.url))
+        res = self._head(path_info.url)
+        if res.status_code == 404:
+            return False
+        if bool(res):
+            return True
+        raise HTTPError(res.status_code, res.reason)
 
-    def get_file_hash(self, path_info):
-        url = path_info.url
-        headers = self.request("HEAD", url).headers
-        etag = headers.get("ETag") or headers.get("Content-MD5")
+    def info(self, path_info):
+        resp = self._head(path_info.url)
+        etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5")
+        size = self._content_length(resp)
+        return {"etag": etag, "size": size}
 
-        if not etag:
-            raise DvcException(
-                "could not find an ETag or "
-                "Content-MD5 header for '{url}'".format(url=url)
-            )
+    def _upload_fobj(self, fobj, to_info):
+        def chunks(fobj):
+            while True:
+                chunk = fobj.read(self.CHUNK_SIZE)
+                if not chunk:
+                    break
+                yield chunk
 
-        if etag.startswith("W/"):
-            raise DvcException(
-                "Weak ETags are not supported."
-                " (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url)
-            )
-
-        return etag
+        response = self.request(self.method, to_info.url, data=chunks(fobj))
+        if response.status_code not in (200, 201):
+            raise HTTPError(response.status_code, response.reason)
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
         response = self.request("GET", from_info.url, stream=True)
@@ -162,30 +182,19 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
                 for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
                     fd_wrapped.write(chunk)
 
-    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
-        def chunks():
-            with open(from_file, "rb") as fd:
-                with Tqdm.wrapattr(
-                    fd,
-                    "read",
-                    total=None
-                    if no_progress_bar
-                    else os.path.getsize(from_file),
-                    leave=False,
-                    desc=to_info.url if name is None else name,
-                    disable=no_progress_bar,
-                ) as fd_wrapped:
-                    while True:
-                        chunk = fd_wrapped.read(self.CHUNK_SIZE)
-                        if not chunk:
-                            break
-                        yield chunk
-
-        response = self.request("POST", to_info.url, data=chunks())
-        if response.status_code not in (200, 201):
-            raise HTTPError(response.status_code, response.reason)
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+    ):
+        with open(from_file, "rb") as fobj:
+            self.upload_fobj(
+                fobj,
+                to_info,
+                no_progress_bar=no_progress_bar,
+                desc=name or to_info.url,
+                total=None if no_progress_bar else os.path.getsize(from_file),
+            )
 
     @staticmethod
-    def _content_length(response):
+    def _content_length(response) -> Optional[int]:
         res = response.headers.get("Content-Length")
         return int(res) if res else None

dvc/fs/https.py

@@ -0,0 +1,7 @@
+from dvc.scheme import Schemes
+
+from .http import HTTPFileSystem
+
+
+class HTTPSFileSystem(HTTPFileSystem):  # pylint:disable=abstract-method
+    scheme = Schemes.HTTPS

dvc/fs/local.py

@@ -0,0 +1,232 @@
+import logging
+import os
+import stat
+
+from funcy import cached_property
+
+from dvc.path_info import PathInfo
+from dvc.scheme import Schemes
+from dvc.system import System
+from dvc.utils import is_exec, tmp_fname
+from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove
+
+from .base import BaseFileSystem
+
+logger = logging.getLogger(__name__)
+
+
+class LocalFileSystem(BaseFileSystem):
+    scheme = Schemes.LOCAL
+    PATH_CLS = PathInfo
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+    TRAVERSE_PREFIX_LEN = 2
+
+    def __init__(self, repo, config, use_dvcignore=False, dvcignore_root=None):
+        super().__init__(repo, config)
+        url = config.get("url")
+        self.path_info = self.PATH_CLS(url) if url else None
+        self.use_dvcignore = use_dvcignore
+        self.dvcignore_root = dvcignore_root
+
+    @property
+    def fs_root(self):
+        return self.config.get("url")
+
+    @cached_property
+    def dvcignore(self):
+        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
+
+        root = self.dvcignore_root or self.fs_root
+        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
+        return cls(self, root)
+
+    @staticmethod
+    def open(path_info, mode="r", encoding=None, **kwargs):
+        return open(path_info, mode=mode, encoding=encoding)
+
+    def exists(self, path_info, use_dvcignore=True):
+        assert isinstance(path_info, str) or path_info.scheme == "local"
+        if self.repo:
+            ret = os.path.lexists(path_info)
+        else:
+            ret = os.path.exists(path_info)
+        if not ret:
+            return False
+        if not use_dvcignore:
+            return True
+
+        return not self.dvcignore.is_ignored_file(
+            path_info
+        ) and not self.dvcignore.is_ignored_dir(path_info)
+
+    def isfile(self, path_info):
+        if not os.path.isfile(path_info):
+            return False
+
+        return not self.dvcignore.is_ignored_file(path_info)
+
+    def isdir(
+        self, path_info, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
+        if not os.path.isdir(path_info):
+            return False
+        return not (use_dvcignore and self.dvcignore.is_ignored_dir(path_info))
+
+    def iscopy(self, path_info):
+        return not (
+            System.is_symlink(path_info) or System.is_hardlink(path_info)
+        )
+
+    def walk(
+        self,
+        top,
+        topdown=True,
+        onerror=None,
+        use_dvcignore=True,
+        ignore_subrepos=True,
+    ):
+        """Directory fs generator.
+
+        See `os.walk` for the docs. Differences:
+        - no support for symlinks
+        """
+        for root, dirs, files in os.walk(
+            top, topdown=topdown, onerror=onerror
+        ):
+            if use_dvcignore:
+                dirs[:], files[:] = self.dvcignore(
+                    os.path.abspath(root),
+                    dirs,
+                    files,
+                    ignore_subrepos=ignore_subrepos,
+                )
+
+            yield os.path.normpath(root), dirs, files
+
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info):
+            for file in files:
+                # NOTE: os.path.join is ~5.5 times slower
+                yield PathInfo(f"{root}{os.sep}{file}")
+
+    def is_empty(self, path_info):
+        if self.isfile(path_info) and os.path.getsize(path_info) == 0:
+            return True
+
+        if self.isdir(path_info) and len(os.listdir(path_info)) == 0:
+            return True
+
+        return False
+
+    def remove(self, path_info):
+        if isinstance(path_info, PathInfo):
+            if path_info.scheme != "local":
+                raise NotImplementedError
+        remove(path_info)
+
+    def makedirs(self, path_info):
+        makedirs(path_info, exist_ok=True)
+
+    def isexec(self, path_info):
+        mode = self.stat(path_info).st_mode
+        return is_exec(mode)
+
+    def stat(self, path):
+        if self.dvcignore.is_ignored(path):
+            raise FileNotFoundError
+
+        return os.stat(path)
+
+    def move(self, from_info, to_info):
+        if from_info.scheme != "local" or to_info.scheme != "local":
+            raise NotImplementedError
+
+        self.makedirs(to_info.parent)
+        move(from_info, to_info)
+
+    def copy(self, from_info, to_info):
+        tmp_info = to_info.parent / tmp_fname("")
+        try:
+            copyfile(from_info, tmp_info)
+            os.rename(tmp_info, to_info)
+        except Exception:
+            self.remove(tmp_info)
+            raise
+
+    def _upload_fobj(self, fobj, to_info):
+        self.makedirs(to_info.parent)
+        tmp_info = to_info.parent / tmp_fname("")
+        try:
+            copy_fobj_to_file(fobj, tmp_info)
+            os.rename(tmp_info, to_info)
+        except Exception:
+            self.remove(tmp_info)
+            raise
+
+    @staticmethod
+    def symlink(from_info, to_info):
+        System.symlink(from_info, to_info)
+
+    @staticmethod
+    def is_symlink(path_info):
+        return System.is_symlink(path_info)
+
+    def hardlink(self, from_info, to_info):
+        # If there are a lot of empty files (which happens a lot in datasets),
+        # and the cache type is `hardlink`, we might reach link limits and
+        # will get something like: `too many links error`
+        #
+        # This is because all those empty files will have the same hash
+        # (i.e. 68b329da9893e34099c7d8ad5cb9c940), therefore, they will be
+        # linked to the same file in the cache.
+        #
+        # From https://en.wikipedia.org/wiki/Hard_link
+        #   * ext4 limits the number of hard links on a file to 65,000
+        #   * Windows with NTFS has a limit of 1024 hard links on a file
+        #
+        # That's why we simply create an empty file rather than a link.
+        if self.getsize(from_info) == 0:
+            self.open(to_info, "w").close()
+
+            logger.debug("Created empty file: %s -> %s", from_info, to_info)
+            return
+
+        System.hardlink(from_info, to_info)
+
+    @staticmethod
+    def is_hardlink(path_info):
+        return System.is_hardlink(path_info)
+
+    def reflink(self, from_info, to_info):
+        System.reflink(from_info, to_info)
+
+    @staticmethod
+    def info(path_info):
+        st = os.stat(path_info)
+        return {
+            "size": st.st_size,
+            "type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
+        }
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs,
+    ):
+        makedirs(to_info.parent, exist_ok=True)
+
+        tmp_file = tmp_fname(to_info)
+        copyfile(
+            from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
+        )
+        os.replace(tmp_file, to_info)
+
+    @staticmethod
+    def _download(
+        from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        copyfile(
+            from_info, to_file, no_progress_bar=no_progress_bar, name=name
+        )
+
+    def _reset(self):
+        return self.__dict__.pop("dvcignore", None)

dvc/fs/memory.py

@@ -0,0 +1,34 @@
+from .base import BaseFileSystem
+
+
+class MemoryFileSystem(BaseFileSystem):
+    scheme = "local"
+    PARAM_CHECKSUM = "md5"
+
+    def __init__(self, repo, config):
+        from fsspec.implementations.memory import MemoryFileSystem as MemFS
+
+        super().__init__(repo, config)
+
+        self.fs = MemFS()
+
+    def exists(self, path_info, use_dvcignore=True):
+        return self.fs.exists(path_info.fspath)
+
+    def open(self, path_info, mode="r", encoding=None, **kwargs):
+        return self.fs.open(
+            path_info.fspath, mode=mode, encoding=encoding, **kwargs
+        )
+
+    def info(self, path_info):
+        return self.fs.info(path_info.fspath)
+
+    def stat(self, path_info):
+        import os
+
+        info = self.fs.info(path_info.fspath)
+
+        return os.stat_result((0, 0, 0, 0, 0, 0, info["size"], 0, 0, 0))
+
+    def walk_files(self, path_info, **kwargs):
+        raise NotImplementedError

dvc/fs/oss.py

@@ -8,12 +8,12 @@ from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
 
-from .base import BaseTree
+from .base import BaseFileSystem
 
 logger = logging.getLogger(__name__)
 
 
-class OSSTree(BaseTree):  # pylint:disable=abstract-method
+class OSSFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     """
     oss2 document:
     https://www.alibabacloud.com/help/doc-detail/32026.htm
@@ -45,6 +45,7 @@ class OSSTree(BaseTree):  # pylint:disable=abstract-method
         url = config.get("url")
         self.path_info = self.PATH_CLS(url) if url else None
 
+        self.bucket = self.path_info.bucket
         self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
 
         self.key_id = (
@@ -64,12 +65,11 @@ class OSSTree(BaseTree):  # pylint:disable=abstract-method
     def oss_service(self):
         import oss2
 
-        logger.debug(f"URL: {self.path_info}")
         logger.debug(f"key id: {self.key_id}")
         logger.debug(f"key secret: {self.key_secret}")
 
         auth = oss2.Auth(self.key_id, self.key_secret)
-        bucket = oss2.Bucket(auth, self.endpoint, self.path_info.bucket)
+        bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
 
         # Ensure bucket exists
         try:
@@ -84,7 +84,7 @@ class OSSTree(BaseTree):  # pylint:disable=abstract-method
         return bucket
 
     def _generate_download_url(self, path_info, expires=3600):
-        assert path_info.bucket == self.path_info.bucket
+        assert path_info.bucket == self.bucket
 
         return self.oss_service.sign_url("GET", path_info.path, expires)
 
@@ -116,6 +116,9 @@ class OSSTree(BaseTree):  # pylint:disable=abstract-method
         logger.debug(f"Removing oss://{path_info}")
         self.oss_service.delete_object(path_info.path)
 
+    def _upload_fobj(self, fobj, to_info):
+        self.oss_service.put_object(to_info.path, fobj)
+
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
@@ -128,6 +131,11 @@ class OSSTree(BaseTree):  # pylint:disable=abstract-method
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
     ):
         with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.oss_service.get_object_to_file(
-                from_info.path, to_file, progress_callback=pbar.update_to
+            import oss2
+
+            oss2.resumable_download(
+                self.oss_service,
+                from_info.path,
+                to_file,
+                progress_callback=pbar.update_to,
             )

dvc/fs/pool.py

@@ -11,7 +11,7 @@ def get_connection(conn_func, *args, **kwargs):
     conn = pool.get_connection()
     try:
         yield conn
-    except BaseException:
+    except Exception:
         conn.close()
         raise
     else:

dvc/fs/repo.py

@@ -0,0 +1,431 @@
+import logging
+import os
+import stat
+import threading
+from contextlib import suppress
+from itertools import takewhile
+from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union
+
+from funcy import lfilter, wrap_with
+
+from dvc.path_info import PathInfo
+
+from .base import BaseFileSystem
+from .dvc import DvcFileSystem
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+logger = logging.getLogger(__name__)
+
+RepoFactory = Union[Callable[[str], "Repo"], Type["Repo"]]
+
+
+class RepoFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
+    """DVC + git-tracked files fs.
+
+    Args:
+        repo: DVC or git repo.
+        subrepos: traverse to subrepos (by default, it ignores subrepos)
+        repo_factory: A function to initialize subrepo with, default is Repo.
+        kwargs: Additional keyword arguments passed to the `DvcFileSystem()`.
+    """
+
+    scheme = "local"
+    PARAM_CHECKSUM = "md5"
+
+    def __init__(
+        self, repo, subrepos=False, repo_factory: RepoFactory = None,
+    ):
+        super().__init__(repo, {"url": repo.root_dir})
+
+        from dvc.utils.collections import PathStringTrie
+
+        if not repo_factory:
+            from dvc.repo import Repo
+
+            self.repo_factory: RepoFactory = Repo
+        else:
+            self.repo_factory = repo_factory
+
+        self._main_repo = repo
+        self.root_dir = repo.root_dir
+        self._traverse_subrepos = subrepos
+
+        self._subrepos_trie = PathStringTrie()
+        """Keeps track of each and every path with the corresponding repo."""
+
+        self._subrepos_trie[self.root_dir] = repo
+
+        self._dvcfss = {}
+        """Keep a dvcfs instance of each repo."""
+
+        if hasattr(repo, "dvc_dir"):
+            self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
+
+    def _get_repo(self, path) -> Optional["Repo"]:
+        """Returns repo that the path falls in, using prefix.
+
+        If the path is already tracked/collected, it just returns the repo.
+
+        Otherwise, it collects the repos that might be in the path's parents
+        and then returns the appropriate one.
+        """
+        repo = self._subrepos_trie.get(path)
+        if repo:
+            return repo
+
+        prefix, repo = self._subrepos_trie.longest_prefix(path)
+        if not prefix:
+            return None
+
+        parents = (parent.fspath for parent in PathInfo(path).parents)
+        dirs = [path] + list(takewhile(lambda p: p != prefix, parents))
+        dirs.reverse()
+        self._update(dirs, starting_repo=repo)
+        return self._subrepos_trie.get(path)
+
+    @wrap_with(threading.Lock())
+    def _update(self, dirs, starting_repo):
+        """Checks for subrepo in directories and updates them."""
+        repo = starting_repo
+        for d in dirs:
+            if self._is_dvc_repo(d):
+                repo = self.repo_factory(
+                    d,
+                    scm=self.repo.scm,
+                    rev=self.repo.get_rev(),
+                    repo_factory=self.repo_factory,
+                )
+                self._dvcfss[repo.root_dir] = DvcFileSystem(repo)
+            self._subrepos_trie[d] = repo
+
+    def _is_dvc_repo(self, dir_path):
+        """Check if the directory is a dvc repo."""
+        if not self._traverse_subrepos:
+            return False
+
+        from dvc.repo import Repo
+
+        repo_path = os.path.join(dir_path, Repo.DVC_DIR)
+        # dvcignore will ignore subrepos, therefore using `use_dvcignore=False`
+        return self._main_repo.fs.isdir(repo_path, use_dvcignore=False)
+
+    def _get_fs_pair(
+        self, path
+    ) -> Tuple[BaseFileSystem, Optional[DvcFileSystem]]:
+        """
+        Returns a pair of fss based on repo the path falls in, using prefix.
+        """
+        path = os.path.abspath(path)
+
+        # fallback to the top-level repo if repo was not found
+        # this can happen if the path is outside of the repo
+        repo = self._get_repo(path) or self._main_repo
+
+        dvc_fs = self._dvcfss.get(repo.root_dir)
+        return repo.fs, dvc_fs
+
+    def open(
+        self, path, mode="r", encoding="utf-8", **kwargs
+    ):  # pylint: disable=arguments-differ
+        if "b" in mode:
+            encoding = None
+
+        fs, dvc_fs = self._get_fs_pair(path)
+        path_info = PathInfo(path)
+        try:
+            return fs.open(path_info, mode=mode, encoding=encoding)
+        except FileNotFoundError:
+            if not dvc_fs:
+                raise
+
+        return dvc_fs.open(path_info, mode=mode, encoding=encoding, **kwargs)
+
+    def exists(
+        self, path, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
+        fs, dvc_fs = self._get_fs_pair(path)
+
+        if not dvc_fs:
+            return fs.exists(path)
+
+        if fs.exists(path):
+            return True
+
+        try:
+            meta = dvc_fs.metadata(path)
+        except FileNotFoundError:
+            return False
+
+        (out,) = meta.outs
+        assert len(meta.outs) == 1
+        if fs.exists(out.path_info):
+            return False
+        return True
+
+    def isdir(self, path):  # pylint: disable=arguments-differ
+        fs, dvc_fs = self._get_fs_pair(path)
+
+        try:
+            st = fs.stat(path)
+            return stat.S_ISDIR(st.st_mode)
+        except (OSError, ValueError):
+            # from CPython's os.path.isdir()
+            pass
+
+        if not dvc_fs:
+            return False
+
+        try:
+            meta = dvc_fs.metadata(path)
+        except FileNotFoundError:
+            return False
+
+        (out,) = meta.outs
+        assert len(meta.outs) == 1
+        if fs.exists(out.path_info):
+            return False
+        return meta.isdir
+
+    def isdvc(self, path, **kwargs):
+        _, dvc_fs = self._get_fs_pair(path)
+        return dvc_fs is not None and dvc_fs.isdvc(path, **kwargs)
+
+    def isfile(self, path):  # pylint: disable=arguments-differ
+        fs, dvc_fs = self._get_fs_pair(path)
+
+        try:
+            st = fs.stat(path)
+            return stat.S_ISREG(st.st_mode)
+        except (OSError, ValueError):
+            # from CPython's os.path.isfile()
+            pass
+
+        if not dvc_fs:
+            return False
+
+        try:
+            meta = dvc_fs.metadata(path)
+        except FileNotFoundError:
+            return False
+
+        (out,) = meta.outs
+        assert len(meta.outs) == 1
+        if fs.exists(out.path_info):
+            return False
+        return meta.isfile
+
+    def isexec(self, path_info):
+        fs, dvc_fs = self._get_fs_pair(path_info)
+        if dvc_fs and dvc_fs.exists(path_info):
+            return dvc_fs.isexec(path_info)
+        return fs.isexec(path_info)
+
+    def stat(self, path):
+        fs, _ = self._get_fs_pair(path)
+        return fs.stat(path)
+
+    def _dvc_walk(self, walk):
+        try:
+            root, dirs, files = next(walk)
+        except StopIteration:
+            return
+        yield root, dirs, files
+        for _ in dirs:
+            yield from self._dvc_walk(walk)
+
+    def _subrepo_walk(self, dir_path, **kwargs):
+        """Walk into a new repo.
+
+         NOTE: subrepo will only be discovered when walking if
+         ignore_subrepos is set to False.
+        """
+        fs, dvc_fs = self._get_fs_pair(dir_path)
+        fs_walk = fs.walk(
+            dir_path, topdown=True, ignore_subrepos=not self._traverse_subrepos
+        )
+        if dvc_fs:
+            dvc_walk = dvc_fs.walk(dir_path, topdown=True, **kwargs)
+        else:
+            dvc_walk = None
+        yield from self._walk(fs_walk, dvc_walk, **kwargs)
+
+    def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False):
+        from dvc.dvcfile import is_valid_filename
+        from dvc.ignore import DvcIgnore
+
+        assert repo_walk
+        try:
+            _, dvc_dirs, dvc_fnames = (
+                next(dvc_walk) if dvc_walk else (None, [], [])
+            )
+            repo_root, repo_dirs, repo_fnames = next(repo_walk)
+        except StopIteration:
+            return
+
+        # separate subdirs into shared dirs, dvc-only dirs, repo-only dirs
+        dvc_set = set(dvc_dirs)
+        repo_set = set(repo_dirs)
+        dvc_only = list(dvc_set - repo_set)
+        repo_only = list(repo_set - dvc_set)
+        shared = list(dvc_set & repo_set)
+        dirs = shared + dvc_only + repo_only
+
+        def _func(fname):
+            if dvcfiles:
+                return True
+
+            return not (
+                is_valid_filename(fname) or fname == DvcIgnore.DVCIGNORE_FILE
+            )
+
+        # merge file lists
+        files = set(filter(_func, dvc_fnames + repo_fnames))
+
+        yield repo_root, dirs, list(files)
+
+        def is_dvc_repo(d):
+            return self._is_dvc_repo(os.path.join(repo_root, d))
+
+        # remove subrepos to prevent it from being traversed
+        subrepos = set(filter(is_dvc_repo, repo_only))
+        # set dir order for next recursion level - shared dirs first so that
+        # next() for both generators recurses into the same shared directory
+        dvc_dirs[:] = [dirname for dirname in dirs if dirname in dvc_set]
+        repo_dirs[:] = lfilter(lambda d: d in (repo_set - subrepos), dirs)
+
+        for dirname in dirs:
+            if dirname in subrepos:
+                dir_path = os.path.join(repo_root, dirname)
+                yield from self._subrepo_walk(dir_path, dvcfiles=dvcfiles)
+            elif dirname in shared:
+                yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
+            elif dirname in dvc_set:
+                yield from self._dvc_walk(dvc_walk)
+            elif dirname in repo_set:
+                yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
+
+    def walk(
+        self,
+        top,
+        topdown=True,
+        onerror=None,
+        dvcfiles=False,
+        follow_subrepos=None,
+        **kwargs
+    ):  # pylint: disable=arguments-differ
+        """Walk and merge both DVC and repo fss.
+
+        Args:
+            top: path to walk from
+            topdown: if True, fs will be walked from top down.
+            onerror: if set, onerror function will be called if an error
+                occurs (by default errors are ignored).
+            dvcfiles: if True, dvcfiles will be included in the files list
+                for walked directories.
+
+        Any kwargs will be passed into methods used for fetching and/or
+        streaming DVC outs from remotes.
+        """
+        assert topdown
+
+        if not self.exists(top):
+            if onerror is not None:
+                onerror(FileNotFoundError(top))
+            return
+
+        if not self.isdir(top):
+            if onerror is not None:
+                onerror(NotADirectoryError(top))
+            return
+
+        ignore_subrepos = not self._traverse_subrepos
+        if follow_subrepos is not None:
+            ignore_subrepos = not follow_subrepos
+
+        fs, dvc_fs = self._get_fs_pair(top)
+        repo_exists = fs.exists(top)
+        repo_walk = fs.walk(
+            top,
+            topdown=topdown,
+            onerror=onerror,
+            ignore_subrepos=ignore_subrepos,
+        )
+
+        if not dvc_fs or (repo_exists and dvc_fs.isdvc(top)):
+            yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
+            return
+
+        if not repo_exists:
+            yield from dvc_fs.walk(top, topdown=topdown, **kwargs)
+
+        dvc_walk = None
+        if dvc_fs.exists(top):
+            dvc_walk = dvc_fs.walk(top, topdown=topdown, **kwargs)
+
+        yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
+
+    def walk_files(self, top, **kwargs):  # pylint: disable=arguments-differ
+        for root, _, files in self.walk(top, **kwargs):
+            for fname in files:
+                yield PathInfo(root) / fname
+
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **kwargs
+    ):
+        import shutil
+
+        from dvc.progress import Tqdm
+
+        with open(to_file, "wb+") as to_fobj:
+            with Tqdm.wrapattr(
+                to_fobj, "write", desc=name, disable=no_progress_bar,
+            ) as wrapped:
+                with self.open(from_info, "rb", **kwargs) as from_fobj:
+                    shutil.copyfileobj(from_fobj, wrapped)
+
+    @property
+    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
+        return self._main_repo.fs.hash_jobs
+
+    def metadata(self, path):
+        abspath = os.path.abspath(path)
+        path_info = PathInfo(abspath)
+        fs, dvc_fs = self._get_fs_pair(path_info)
+
+        dvc_meta = None
+        if dvc_fs:
+            with suppress(FileNotFoundError):
+                dvc_meta = dvc_fs.metadata(path_info)
+
+        stat_result = None
+        with suppress(FileNotFoundError):
+            stat_result = fs.stat(path_info)
+
+        if not stat_result and not dvc_meta:
+            raise FileNotFoundError
+
+        from ._metadata import Metadata
+
+        meta = dvc_meta or Metadata(
+            path_info=path_info,
+            repo=self._get_repo(abspath) or self._main_repo,
+        )
+
+        isdir = bool(stat_result) and stat.S_ISDIR(stat_result.st_mode)
+        meta.isdir = meta.isdir or isdir
+
+        if not dvc_meta:
+            from dvc.utils import is_exec
+
+            meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode)
+        return meta
+
+    def info(self, path_info):
+        fs, dvc_fs = self._get_fs_pair(path_info)
+
+        try:
+            return fs.info(path_info)
+        except FileNotFoundError:
+            return dvc_fs.info(path_info)

dvc/fs/s3.py

@@ -1,25 +1,31 @@
 import logging
 import os
 import threading
+from contextlib import contextmanager
 
 from funcy import cached_property, wrap_prop
 
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException, ETagMismatchError
+from dvc.hash_info import HashInfo
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
+from dvc.utils import conversions, error_link
 
-from .base import BaseTree
+from .base import BaseFileSystem
 
 logger = logging.getLogger(__name__)
 
+_AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
 
-class S3Tree(BaseTree):
+
+class S3FileSystem(BaseFileSystem):
     scheme = Schemes.S3
     PATH_CLS = CloudURLInfo
     REQUIRES = {"boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
+    DETAIL_FIELDS = frozenset(("etag", "size"))
 
     def __init__(self, repo, config):
         super().__init__(repo, config)
@@ -31,11 +37,6 @@ class S3Tree(BaseTree):
         self.profile = config.get("profile")
         self.endpoint_url = config.get("endpointurl")
 
-        if config.get("listobjects"):
-            self.list_objects_api = "list_objects"
-        else:
-            self.list_objects_api = "list_objects_v2"
-
         self.use_ssl = config.get("use_ssl", True)
 
         self.extra_args = {}
@@ -56,43 +57,130 @@ class S3Tree(BaseTree):
 
         self.access_key_id = config.get("access_key_id")
         self.secret_access_key = config.get("secret_access_key")
+        self.session_token = config.get("session_token")
 
         shared_creds = config.get("credentialpath")
         if shared_creds:
             os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
+        config_path = config.get("configpath")
+        if config_path:
+            os.environ.setdefault("AWS_CONFIG_FILE", config_path)
+        self._transfer_config = None
+
+    # https://github.com/aws/aws-cli/blob/0376c6262d6b15dc36c82e6da6e1aad10249cc8c/awscli/customizations/s3/transferconfig.py#L107-L113
+    _TRANSFER_CONFIG_ALIASES = {
+        "max_queue_size": "max_io_queue",
+        "max_concurrent_requests": "max_concurrency",
+        "multipart_threshold": "multipart_threshold",
+        "multipart_chunksize": "multipart_chunksize",
+    }
+
+    def _transform_config(self, s3_config):
+        """Splits the general s3 config into 2 different config
+        objects, one for transfer.TransferConfig and other is the
+        general session config"""
+
+        config, transfer_config = {}, {}
+        for key, value in s3_config.items():
+            if key in self._TRANSFER_CONFIG_ALIASES:
+                if key in {"multipart_chunksize", "multipart_threshold"}:
+                    # cast human readable sizes (like 24MiB) to integers
+                    value = conversions.human_readable_to_bytes(value)
+                else:
+                    value = int(value)
+                transfer_config[self._TRANSFER_CONFIG_ALIASES[key]] = value
+            else:
+                config[key] = value
+
+        return config, transfer_config
+
+    def _process_config(self):
+        from boto3.s3.transfer import TransferConfig
+        from botocore.configloader import load_config
+
+        config_path = os.environ.get("AWS_CONFIG_FILE", _AWS_CONFIG_PATH)
+        if not os.path.exists(config_path):
+            return None
+
+        config = load_config(config_path)
+        profile = config["profiles"].get(self.profile or "default")
+        if not profile:
+            return None
+
+        s3_config = profile.get("s3", {})
+        s3_config, transfer_config = self._transform_config(s3_config)
+        self._transfer_config = TransferConfig(**transfer_config)
+        return s3_config
+
     @wrap_prop(threading.Lock())
     @cached_property
     def s3(self):
         import boto3
 
-        session_opts = dict(profile_name=self.profile, region_name=self.region)
+        session_opts = {
+            "profile_name": self.profile,
+            "region_name": self.region,
+        }
 
         if self.access_key_id:
             session_opts["aws_access_key_id"] = self.access_key_id
         if self.secret_access_key:
             session_opts["aws_secret_access_key"] = self.secret_access_key
+        if self.session_token:
+            session_opts["aws_session_token"] = self.session_token
 
         session = boto3.session.Session(**session_opts)
+        s3_config = self._process_config()
 
-        return session.client(
-            "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl
+        return session.resource(
+            "s3",
+            endpoint_url=self.endpoint_url,
+            use_ssl=self.use_ssl,
+            config=boto3.session.Config(
+                signature_version="s3v4", s3=s3_config
+            ),
         )
 
-    @classmethod
-    def get_etag(cls, s3, bucket, path):
-        obj = cls.get_head_object(s3, bucket, path)
-
-        return obj["ETag"].strip('"')
-
-    @staticmethod
-    def get_head_object(s3, bucket, path, *args, **kwargs):
+    @contextmanager
+    def _get_s3(self):
+        from botocore.exceptions import (
+            EndpointConnectionError,
+            NoCredentialsError,
+        )
 
         try:
-            obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
-        except Exception as exc:
-            raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
-        return obj
+            yield self.s3
+        except NoCredentialsError as exc:
+            link = error_link("no-credentials")
+            raise DvcException(
+                f"Unable to find AWS credentials. {link}"
+            ) from exc
+        except EndpointConnectionError as exc:
+            link = error_link("connection-error")
+            name = self.endpoint_url or "AWS S3"
+            raise DvcException(
+                f"Unable to connect to '{name}'. {link}"
+            ) from exc
+
+    @contextmanager
+    def _get_bucket(self, bucket):
+        with self._get_s3() as s3:
+            try:
+                yield s3.Bucket(bucket)
+            except s3.meta.client.exceptions.NoSuchBucket as exc:
+                link = error_link("no-bucket")
+                raise DvcException(
+                    f"Bucket '{bucket}' does not exist. {link}"
+                ) from exc
+
+    @contextmanager
+    def _get_obj(self, path_info):
+        with self._get_bucket(path_info.bucket) as bucket:
+            try:
+                yield bucket.Object(path_info.path)
+            except bucket.meta.client.exceptions.NoSuchKey as exc:
+                raise DvcException(f"{path_info.url} does not exist") from exc
 
     def _append_aws_grants_to_extra_args(self, config):
         # Keys for extra_args can be one of the following list:
@@ -127,9 +215,12 @@ class S3Tree(BaseTree):
 
     def _generate_download_url(self, path_info, expires=3600):
         params = {"Bucket": path_info.bucket, "Key": path_info.path}
-        return self.s3.generate_presigned_url(
-            ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
-        )
+        with self._get_s3() as s3:
+            return s3.meta.client.generate_presigned_url(
+                ClientMethod="get_object",
+                Params=params,
+                ExpiresIn=int(expires),
+            )
 
     def exists(self, path_info, use_dvcignore=True):
         """Check if the blob exists. If it does not exist,
@@ -164,36 +255,32 @@ class S3Tree(BaseTree):
         return bool(list(self._list_paths(dir_path, max_items=1)))
 
     def isfile(self, path_info):
-        from botocore.exceptions import ClientError
-
         if path_info.path.endswith("/"):
             return False
 
-        try:
-            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
-        except ClientError as exc:
-            if exc.response["Error"]["Code"] != "404":
-                raise
-            return False
+        return path_info.path in self._list_paths(path_info)
 
-        return True
+    def info(self, path_info):
+        # temporary workaround, will be removed when migrating to s3fs
+        if self.isdir(path_info):
+            return {"type": "directory"}
 
-    def _list_objects(self, path_info, max_items=None):
-        """ Read config for list object api, paginate through list objects."""
-        kwargs = {
-            "Bucket": path_info.bucket,
-            "Prefix": path_info.path,
-            "PaginationConfig": {"MaxItems": max_items},
-        }
-        paginator = self.s3.get_paginator(self.list_objects_api)
-        for page in paginator.paginate(**kwargs):
-            contents = page.get("Contents", ())
-            yield from contents
+        with self._get_obj(path_info) as obj:
+            return {
+                "type": "file",
+                "etag": obj.e_tag.strip('"'),
+                "size": obj.content_length,
+            }
 
     def _list_paths(self, path_info, max_items=None):
-        return (
-            item["Key"] for item in self._list_objects(path_info, max_items)
-        )
+        with self._get_bucket(path_info.bucket) as bucket:
+            obj_summaries = bucket.objects.filter(Prefix=path_info.path)
+            if max_items is not None:
+                obj_summaries = obj_summaries.page_size(max_items).limit(
+                    max_items
+                )
+            for obj_summary in obj_summaries:
+                yield obj_summary.key
 
     def walk_files(self, path_info, **kwargs):
         if not kwargs.pop("prefix", False):
@@ -204,12 +291,30 @@ class S3Tree(BaseTree):
 
             yield path_info.replace(path=fname)
 
+    def ls(
+        self, path_info, detail=False, recursive=False
+    ):  # pylint: disable=arguments-differ
+        assert recursive
+
+        with self._get_bucket(path_info.bucket) as bucket:
+            for obj_summary in bucket.objects.filter(Prefix=path_info.path):
+                if detail:
+                    yield {
+                        "type": "file",
+                        "name": obj_summary.key,
+                        "size": obj_summary.size,
+                        "etag": obj_summary.e_tag.strip('"'),
+                    }
+                else:
+                    yield obj_summary.key
+
     def remove(self, path_info):
         if path_info.scheme != "s3":
             raise NotImplementedError
 
         logger.debug(f"Removing {path_info}")
-        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
+        with self._get_obj(path_info) as obj:
+            obj.delete()
 
     def makedirs(self, path_info):
         # We need to support creating empty directories, which means
@@ -221,10 +326,12 @@ class S3Tree(BaseTree):
             return
 
         dir_path = path_info / ""
-        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
+        with self._get_obj(dir_path) as obj:
+            obj.put(Body="")
 
     def copy(self, from_info, to_info):
-        self._copy(self.s3, from_info, to_info, self.extra_args)
+        with self._get_s3() as s3:
+            self._copy(s3.meta.client, from_info, to_info, self.extra_args)
 
     @classmethod
     def _copy_multipart(
@@ -238,8 +345,8 @@ class S3Tree(BaseTree):
         parts = []
         byte_position = 0
         for i in range(1, n_parts + 1):
-            obj = S3Tree.get_head_object(
-                s3, from_info.bucket, from_info.path, PartNumber=i
+            obj = s3.head_object(
+                Bucket=from_info.bucket, Key=from_info.path, PartNumber=i
             )
             part_size = obj["ContentLength"]
             lastbyte = byte_position + part_size - 1
@@ -293,7 +400,7 @@ class S3Tree(BaseTree):
         # object is transfered in the same chunks as it was originally.
         from boto3.s3.transfer import TransferConfig
 
-        obj = S3Tree.get_head_object(s3, from_info.bucket, from_info.path)
+        obj = s3.head_object(Bucket=from_info.bucket, Key=from_info.path)
         etag = obj["ETag"].strip('"')
         size = obj["ContentLength"]
 
@@ -313,36 +420,43 @@ class S3Tree(BaseTree):
                 Config=TransferConfig(multipart_threshold=size + 1),
             )
 
-        cached_etag = S3Tree.get_etag(s3, to_info.bucket, to_info.path)
+        cached_etag = s3.head_object(Bucket=to_info.bucket, Key=to_info.path)[
+            "ETag"
+        ].strip('"')
         if etag != cached_etag:
             raise ETagMismatchError(etag, cached_etag)
 
-    def get_file_hash(self, path_info):
-        return self.get_etag(self.s3, path_info.bucket, path_info.path)
+    def etag(self, path_info):
+        with self._get_obj(path_info) as obj:
+            return HashInfo("etag", size=obj.content_length,)
 
-    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
-        total = os.path.getsize(from_file)
-        with Tqdm(
-            disable=no_progress_bar, total=total, bytes=True, desc=name
-        ) as pbar:
-            self.s3.upload_file(
-                from_file,
-                to_info.bucket,
-                to_info.path,
-                Callback=pbar.update,
-                ExtraArgs=self.extra_args,
-            )
+    def _upload_fobj(self, fobj, to_info):
+        with self._get_obj(to_info) as obj:
+            obj.upload_fileobj(fobj, Config=self._transfer_config)
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+    ):
+        with self._get_obj(to_info) as obj:
+            total = os.path.getsize(from_file)
+            with Tqdm(
+                disable=no_progress_bar, total=total, bytes=True, desc=name
+            ) as pbar:
+                obj.upload_file(
+                    from_file,
+                    Callback=pbar.update,
+                    ExtraArgs=self.extra_args,
+                    Config=self._transfer_config,
+                )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        if no_progress_bar:
-            total = None
-        else:
-            total = self.s3.head_object(
-                Bucket=from_info.bucket, Key=from_info.path
-            )["ContentLength"]
-        with Tqdm(
-            disable=no_progress_bar, total=total, bytes=True, desc=name
-        ) as pbar:
-            self.s3.download_file(
-                from_info.bucket, from_info.path, to_file, Callback=pbar.update
-            )
+        with self._get_obj(from_info) as obj:
+            with Tqdm(
+                disable=no_progress_bar,
+                total=obj.content_length,
+                bytes=True,
+                desc=name,
+            ) as pbar:
+                obj.download_file(
+                    to_file, Callback=pbar.update, Config=self._transfer_config
+                )

dvc/fs/ssh/__init__.py

@@ -2,7 +2,7 @@ import getpass
 import io
 import logging
 import os
-import posixpath
+import shutil
 import threading
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse
@@ -10,9 +10,10 @@ from urllib.parse import urlparse
 from funcy import first, memoize, silent, wrap_with
 
 import dvc.prompt as prompt
+from dvc.hash_info import HashInfo
 from dvc.scheme import Schemes
 
-from ..base import BaseTree
+from ..base import BaseFileSystem
 from ..pool import get_connection
 
 logger = logging.getLogger(__name__)
@@ -29,7 +30,7 @@ def ask_password(host, user, port):
     )
 
 
-class SSHTree(BaseTree):
+class SSHFileSystem(BaseFileSystem):
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
     JOBS = 4
@@ -90,6 +91,7 @@ class SSHTree(BaseTree):
             self.sock = paramiko.ProxyCommand(proxy_command)
         else:
             self.sock = None
+        self.allow_agent = config.get("allow_agent", True)
 
     @staticmethod
     def ssh_config_filename():
@@ -99,7 +101,7 @@ class SSHTree(BaseTree):
     def _load_user_ssh_config(hostname):
         import paramiko
 
-        user_config_file = SSHTree.ssh_config_filename()
+        user_config_file = SSHFileSystem.ssh_config_filename()
         user_ssh_config = {}
         if hostname and os.path.exists(user_config_file):
             ssh_config = paramiko.SSHConfig()
@@ -142,10 +144,11 @@ class SSHTree(BaseTree):
             password=self.password,
             gss_auth=self.gss_auth,
             sock=self.sock,
+            allow_agent=self.allow_agent,
         )
 
     @contextmanager
-    def open(self, path_info, mode="r", encoding=None):
+    def open(self, path_info, mode="r", encoding=None, **kwargs):
         assert mode in {"r", "rt", "rb", "wb"}
 
         with self.ssh(path_info) as ssh, closing(
@@ -184,8 +187,7 @@ class SSHTree(BaseTree):
         with self.ssh(path_info) as ssh:
             ssh.makedirs(path_info.path)
 
-    def move(self, from_info, to_info, mode=None):
-        assert mode is None
+    def move(self, from_info, to_info):
         if from_info.scheme != self.scheme or to_info.scheme != self.scheme:
             raise NotImplementedError
 
@@ -233,20 +235,25 @@ class SSHTree(BaseTree):
         with self.ssh(from_info) as ssh:
             ssh.reflink(from_info.path, to_info.path)
 
-    def get_file_hash(self, path_info):
-        if path_info.scheme != self.scheme:
-            raise NotImplementedError
-
+    def md5(self, path_info):
         with self.ssh(path_info) as ssh:
-            return ssh.md5(path_info.path)
+            return HashInfo(
+                "md5",
+                ssh.md5(path_info.path),
+                size=ssh.getsize(path_info.path),
+            )
 
-    def getsize(self, path_info):
+    def info(self, path_info):
         with self.ssh(path_info) as ssh:
-            return ssh.getsize(path_info.path)
+            return ssh.info(path_info.path)
+
+    def _upload_fobj(self, fobj, to_info):
+        self.makedirs(to_info.parent)
+        with self.open(to_info, mode="wb") as fdest:
+            shutil.copyfileobj(fobj, fdest)
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        assert from_info.isin(self.path_info)
-        with self.ssh(self.path_info) as ssh:
+        with self.ssh(from_info) as ssh:
             ssh.download(
                 from_info.path,
                 to_file,
@@ -254,28 +261,13 @@ class SSHTree(BaseTree):
                 no_progress_bar=no_progress_bar,
             )
 
-    def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
-        assert to_info.isin(self.path_info)
-        with self.ssh(self.path_info) as ssh:
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+    ):
+        with self.ssh(to_info) as ssh:
             ssh.upload(
                 from_file,
                 to_info.path,
                 progress_title=name,
                 no_progress_bar=no_progress_bar,
             )
-
-    def list_paths(self, prefix=None, progress_callback=None):
-        if prefix:
-            root = posixpath.join(self.path_info.path, prefix[:2])
-        else:
-            root = self.path_info.path
-        with self.ssh(self.path_info) as ssh:
-            if prefix and not ssh.exists(root):
-                return
-            # If we simply return an iterator then with above closes instantly
-            if progress_callback:
-                for path in ssh.walk_files(root):
-                    progress_callback()
-                    yield path
-            else:
-                yield from ssh.walk_files(root)

dvc/fs/ssh/connection.py

@@ -2,6 +2,7 @@ import errno
 import logging
 import os
 import posixpath
+import shlex
 import stat
 from contextlib import suppress
 
@@ -39,7 +40,9 @@ class SSHConnection:
                 host=host, **kwargs
             )
         )
-        self.timeout = kwargs.get("timeout", 1800)
+
+        kwargs.setdefault("timeout", 1800)
+        self.timeout = kwargs["timeout"]
 
         self._ssh = paramiko.SSHClient()
 
@@ -69,16 +72,21 @@ class SSHConnection:
         self._ssh.close()
 
     def st_mode(self, path):
+        lstat = None
         with suppress(FileNotFoundError):
-            return self.sftp.lstat(path).st_mode
+            lstat = self.sftp.lstat(path)
 
-        return 0
+        return lstat.st_mode if lstat else 0
+
+    def info(self, path):
+        st = self.sftp.stat(path)
+        return {
+            "size": st.st_size,
+            "type": "dir" if stat.S_ISDIR(st.st_mode) else "file",
+        }
 
     def getsize(self, path):
-        with suppress(FileNotFoundError):
-            return self.sftp.lstat(path).st_size
-
-        return 0
+        return self.info(path)["size"]
 
     def exists(self, path):
         return bool(self.st_mode(path))
@@ -186,6 +194,7 @@ class SSHConnection:
             desc=progress_title or os.path.basename(src),
             disable=no_progress_bar,
             bytes=True,
+            total=self.getsize(src),
         ) as pbar:
             self.sftp.get(src, dest, callback=pbar.update_to)
 
@@ -218,7 +227,10 @@ class SSHConnection:
             progress_title = posixpath.basename(dest)
 
         with Tqdm(
-            desc=progress_title, disable=no_progress_bar, bytes=True
+            desc=progress_title,
+            disable=no_progress_bar,
+            bytes=True,
+            total=os.path.getsize(src),
         ) as pbar:
             self.sftp.put(src, tmp_file, callback=pbar.update_to)
 
@@ -291,6 +303,7 @@ class SSHConnection:
          Example:
               MD5 (foo.txt) = f3d220a856b52aabbf294351e8a24300
         """
+        path = shlex.quote(path)
         if self.uname == "Linux":
             md5 = self.execute("md5sum " + path).split()[0]
         elif self.uname == "Darwin":
@@ -304,6 +317,8 @@ class SSHConnection:
         return md5
 
     def copy(self, src, dest):
+        dest = shlex.quote(dest)
+        src = shlex.quote(src)
         self.execute(f"cp {src} {dest}")
 
     def open_max_sftp_channels(self):
@@ -325,6 +340,8 @@ class SSHConnection:
         self.sftp.symlink(src, dest)
 
     def reflink(self, src, dest):
+        dest = shlex.quote(dest)
+        src = shlex.quote(src)
         if self.uname == "Linux":
             return self.execute(f"cp --reflink {src} {dest}")
 
@@ -334,4 +351,6 @@ class SSHConnection:
         raise DvcException(f"'{self.uname}' is not supported as a SSH remote")
 
     def hardlink(self, src, dest):
+        dest = shlex.quote(dest)
+        src = shlex.quote(src)
         self.execute(f"ln {src} {dest}")

dvc/fs/webdav.py

@@ -0,0 +1,284 @@
+import io
+import logging
+import os
+import threading
+from collections import deque
+
+from funcy import cached_property, nullcontext, wrap_prop
+
+from dvc.config import ConfigError
+from dvc.exceptions import DvcException
+from dvc.path_info import HTTPURLInfo, WebDAVURLInfo
+from dvc.progress import Tqdm
+from dvc.scheme import Schemes
+
+from .base import BaseFileSystem
+from .http import ask_password
+
+logger = logging.getLogger(__name__)
+
+
+class WebDAVConnectionError(DvcException):
+    def __init__(self, host):
+        super().__init__(f"Unable to connect to WebDAV {host}.")
+
+
+class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
+    # Use webdav scheme
+    scheme = Schemes.WEBDAV
+
+    # URLInfo for Webdav ~ replaces webdav -> http
+    PATH_CLS = WebDAVURLInfo
+
+    # Traversable as walk_files is implemented
+    CAN_TRAVERSE = True
+
+    # Length of walk_files prefix
+    TRAVERSE_PREFIX_LEN = 2
+
+    # Implementation based on webdav3.client
+    REQUIRES = {"webdavclient3": "webdav3.client"}
+
+    # Chunk size for buffered upload/download with progress bar
+    CHUNK_SIZE = 2 ** 16
+
+    PARAM_CHECKSUM = "etag"
+    DETAIL_FIELDS = frozenset(("etag", "size"))
+
+    # Constructor
+    def __init__(self, repo, config):
+        # Call BaseFileSystem constructor
+        super().__init__(repo, config)
+
+        # Get username from configuration
+        self.user = config.get("user", None)
+
+        # Get password from configuration (might be None ~ not set)
+        self.password = config.get("password", None)
+
+        # Whether to ask for password if it is not set
+        self.ask_password = config.get("ask_password", False)
+
+        # Use token for webdav auth
+        self.token = config.get("token", None)
+
+        # Path to certificate
+        self.cert_path = config.get("cert_path", None)
+
+        # Path to private key
+        self.key_path = config.get("key_path", None)
+
+        # Connection timeout
+        self.timeout = config.get("timeout", 30)
+
+        # Get URL from configuration
+        self.url = config.get("url", None)
+
+        # If URL in config parse path_info
+        if self.url:
+            self.path_info = self.PATH_CLS(self.url)
+
+            # If username not specified try to use from URL
+            if self.user is None and self.path_info.user is not None:
+                self.user = self.path_info.user
+
+            # Construct hostname from path_info by stripping path
+            http_info = HTTPURLInfo(self.path_info.url)
+            self.hostname = http_info.replace(path="").url
+
+    # Webdav client
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def _client(self):
+        from webdav3.client import Client
+
+        # Set password or ask for it
+        if self.ask_password and self.password is None and self.token is None:
+            self.password = ask_password(self.hostname, self.user)
+
+        # Setup webdav client options dictionary
+        options = {
+            "webdav_hostname": self.hostname,
+            "webdav_login": self.user,
+            "webdav_password": self.password,
+            "webdav_token": self.token,
+            "webdav_cert_path": self.cert_path,
+            "webdav_key_path": self.key_path,
+            "webdav_timeout": self.timeout,
+            "webdav_chunk_size": self.CHUNK_SIZE,
+        }
+
+        client = Client(options)
+
+        # Check whether client options are valid
+        if not client.valid():
+            raise ConfigError(
+                f"Configuration for WebDAV {self.hostname} is invalid."
+            )
+
+        # Check whether connection is valid (root should always exist)
+        if not client.check("/"):
+            raise WebDAVConnectionError(self.hostname)
+
+        return client
+
+    def open(self, path_info, mode="r", encoding=None, **kwargs):
+        from webdav3.exceptions import RemoteResourceNotFound
+
+        assert mode in {"r", "rt", "rb"}
+
+        fobj = io.BytesIO()
+
+        try:
+            self._client.download_from(buff=fobj, remote_path=path_info.path)
+        except RemoteResourceNotFound as exc:
+            raise FileNotFoundError from exc
+
+        fobj.seek(0)
+
+        if "mode" == "rb":
+            return fobj
+
+        return io.TextIOWrapper(fobj, encoding=encoding)
+
+    # Checks whether file/directory exists at remote
+    def exists(self, path_info, use_dvcignore=True):
+        # Use webdav check to test for file existence
+        return self._client.check(path_info.path)
+
+    # Checks whether path points to directory
+    def isdir(self, path_info):
+        # Use webdav is_dir to test whether path points to a directory
+        return self._client.is_dir(path_info.path)
+
+    # Yields path info to all files
+    def walk_files(self, path_info, **kwargs):
+        # Check whether directory exists
+        if not self.exists(path_info):
+            return
+
+        # Collect directories
+        dirs = deque([path_info.path])
+
+        # Iterate all directories found so far
+        while dirs:
+            # Iterate directory content
+            for entry in self._client.list(dirs.pop(), get_info=True):
+                # Construct path_info to entry
+                info = path_info.replace(path=entry["path"])
+
+                # Check whether entry is a directory
+                if entry["isdir"]:
+                    # Append new found directory to directory list
+                    dirs.append(info.path)
+                else:
+                    # Yield path info to non directory
+                    yield info
+
+    def ls(
+        self, path_info, detail=False, recursive=False
+    ):  # pylint: disable=arguments-differ
+        dirs = deque([path_info.path])
+
+        while dirs:
+            for entry in self._client.list(dirs.pop(), get_info=True):
+                path = entry["path"]
+                if entry["isdir"]:
+                    dirs.append(path)
+                    continue
+
+                if detail:
+                    yield {
+                        "type": "file",
+                        "name": path,
+                        "size": entry["size"],
+                        "etag": entry["etag"],
+                    }
+                else:
+                    yield path
+
+            if not recursive:
+                for entry in dirs:
+                    if detail:
+                        yield {"type": "directory", "name": entry}
+                    else:
+                        yield entry
+                return None
+
+    # Removes file/directory
+    def remove(self, path_info):
+        # Use webdav client clean (DELETE) method to remove file/directory
+        self._client.clean(path_info.path)
+
+    # Creates directories
+    def makedirs(self, path_info):
+        # Terminate recursion
+        if path_info.path == "/" or self.exists(path_info):
+            return
+
+        # Recursively descent to root
+        self.makedirs(path_info.parent)
+
+        # Construct directory at current recursion depth
+        self._client.mkdir(path_info.path)
+
+    # Moves file/directory at remote
+    def move(self, from_info, to_info):
+        # Webdav client move
+        self._client.move(from_info.path, to_info.path)
+
+    def _upload_fobj(self, fobj, to_info):
+        # In contrast to other upload_fobj implementations, this one does not
+        # exactly do a chunked-upload but rather put everything in one request.
+        self.makedirs(to_info.parent)
+        self._client.upload_to(buff=fobj, remote_path=to_info.path)
+
+    # Downloads file from remote to file
+    def _download(self, from_info, to_file, name=None, no_progress_bar=False):
+        # Progress from HTTPFileSystem
+        with open(to_file, "wb") as fd:
+            with Tqdm.wrapattr(
+                fd,
+                "write",
+                total=None if no_progress_bar else self.getsize(from_info),
+                leave=False,
+                desc=from_info.url if name is None else name,
+                disable=no_progress_bar,
+            ) as fd_wrapped:
+                # Download from WebDAV via buffer
+                self._client.download_from(
+                    buff=fd_wrapped, remote_path=from_info.path
+                )
+
+    # Uploads file to remote
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs,
+    ):
+        # First try to create parent directories
+        self.makedirs(to_info.parent)
+
+        file_size = os.path.getsize(from_file)
+        with open(from_file, "rb") as fd:
+            progress_context = (
+                nullcontext(fd)
+                if file_size == 0
+                else Tqdm.wrapattr(
+                    fd,
+                    "read",
+                    total=None if no_progress_bar else file_size,
+                    leave=False,
+                    desc=to_info.url if name is None else name,
+                    disable=no_progress_bar,
+                )
+            )
+            with progress_context as fd_wrapped:
+                self._client.upload_to(
+                    buff=fd_wrapped, remote_path=to_info.path
+                )
+
+    def info(self, path_info):
+        info = self._client.info(path_info.path)
+        return {
+            "size": int(info["size"]),
+            "etag": info["etag"],
+        }

dvc/fs/webdavs.py

@@ -0,0 +1,7 @@
+from dvc.scheme import Schemes
+
+from .webdav import WebDAVFileSystem
+
+
+class WebDAVSFileSystem(WebDAVFileSystem):  # pylint:disable=abstract-method
+    scheme = Schemes.WEBDAVS

dvc/fs/webhdfs.py

@@ -0,0 +1,171 @@
+import logging
+import os
+import posixpath
+import shutil
+import threading
+from contextlib import contextmanager
+from urllib.parse import urlparse
+
+from funcy import cached_property, wrap_prop
+
+from dvc.hash_info import HashInfo
+from dvc.path_info import CloudURLInfo
+from dvc.progress import Tqdm
+from dvc.scheme import Schemes
+
+from .base import BaseFileSystem
+
+logger = logging.getLogger(__name__)
+
+
+def update_pbar(pbar, total):
+    """Update pbar to accept the two arguments passed by hdfs"""
+
+    def update(_, bytes_transfered):
+        if bytes_transfered == -1:
+            pbar.update_to(total)
+            return
+        pbar.update_to(bytes_transfered)
+
+    return update
+
+
+class WebHDFSFileSystem(BaseFileSystem):
+    scheme = Schemes.WEBHDFS
+    PATH_CLS = CloudURLInfo
+    REQUIRES = {"hdfs": "hdfs"}
+    PARAM_CHECKSUM = "checksum"
+    TRAVERSE_PREFIX_LEN = 2
+
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+
+        self.path_info = None
+        url = config.get("url")
+        if not url:
+            return
+
+        self.path_info = self.PATH_CLS(url)
+
+        parsed = urlparse(url)
+
+        self.host = parsed.hostname
+        self.user = parsed.username or config.get("user")
+        self.port = parsed.port
+
+        self.hdfscli_config = config.get("hdfscli_config")
+        self.token = config.get("webhdfs_token")
+        self.alias = config.get("webhdfs_alias")
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def hdfs_client(self):
+        import hdfs
+
+        logger.debug("HDFSConfig: %s", self.hdfscli_config)
+
+        try:
+            return hdfs.config.Config(self.hdfscli_config).get_client(
+                self.alias
+            )
+        except hdfs.util.HdfsError as exc:
+            exc_msg = str(exc)
+            errors = (
+                "No alias specified",
+                "Invalid configuration file",
+                f"Alias {self.alias} not found",
+            )
+            if not any(err in exc_msg for err in errors):
+                raise
+
+            http_url = f"http://{self.host}:{self.port}"
+            logger.debug("URL: %s", http_url)
+
+            if self.token is not None:
+                client = hdfs.TokenClient(http_url, token=self.token, root="/")
+            else:
+                client = hdfs.InsecureClient(
+                    http_url, user=self.user, root="/"
+                )
+
+        return client
+
+    @contextmanager
+    def open(self, path_info, mode="r", encoding=None, **kwargs):
+        assert mode in {"r", "rt", "rb"}
+
+        with self.hdfs_client.read(
+            path_info.path, encoding=encoding
+        ) as reader:
+            yield reader
+
+    def walk_files(self, path_info, **kwargs):
+        if not self.exists(path_info):
+            return
+
+        root = path_info.path
+        for path, _, fnames in self.hdfs_client.walk(root):
+            for fname in fnames:
+                yield path_info.replace(path=posixpath.join(path, fname))
+
+    def remove(self, path_info):
+        if path_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        self.hdfs_client.delete(path_info.path)
+
+    def exists(self, path_info, use_dvcignore=True):
+        assert not isinstance(path_info, list)
+        assert path_info.scheme == "webhdfs"
+
+        status = self.hdfs_client.status(path_info.path, strict=False)
+        return status is not None
+
+    def info(self, path_info):
+        st = self.hdfs_client.status(path_info.path)
+        return {"size": st["length"]}
+
+    def checksum(self, path_info):
+        return HashInfo(
+            "checksum",
+            self.hdfs_client.checksum(path_info.path)["bytes"],
+            size=self.hdfs_client.status(path_info.path)["length"],
+        )
+
+    def copy(self, from_info, to_info, **_kwargs):
+        with self.hdfs_client.read(from_info.path) as reader:
+            with self.hdfs_client.write(to_info.path) as writer:
+                shutil.copyfileobj(reader, writer)
+
+    def move(self, from_info, to_info):
+        self.hdfs_client.makedirs(to_info.parent.path)
+        self.hdfs_client.rename(from_info.path, to_info.path)
+
+    def _upload_fobj(self, fobj, to_info):
+        with self.hdfs_client.write(to_info.path) as fdest:
+            shutil.copyfileobj(fobj, fdest)
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+    ):
+        total = os.path.getsize(from_file)
+        with Tqdm(
+            desc=name, total=total, disable=no_progress_bar, bytes=True
+        ) as pbar:
+            self.hdfs_client.upload(
+                to_info.path,
+                from_file,
+                overwrite=True,
+                progress=update_pbar(pbar, total),
+            )
+
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
+    ):
+        total = self.getsize(from_info)
+        with Tqdm(
+            desc=name, total=total, disable=no_progress_bar, bytes=True
+        ) as pbar:
+            self.hdfs_client.download(
+                from_info.path, to_file, progress=update_pbar(pbar, total)
+            )

dvc/hash_info.py

@@ -0,0 +1,52 @@
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from typing import Optional
+
+HASH_DIR_SUFFIX = ".dir"
+
+
+@dataclass
+class HashInfo:
+    PARAM_SIZE = "size"
+    PARAM_NFILES = "nfiles"
+
+    name: Optional[str]
+    value: Optional[str]
+    size: Optional[int] = field(default=None, compare=False)
+    nfiles: Optional[int] = field(default=None, compare=False)
+
+    def __bool__(self):
+        return bool(self.value)
+
+    def __str__(self):
+        return f"{self.name}: {self.value}"
+
+    @classmethod
+    def from_dict(cls, d):
+        _d = d.copy() if d else {}
+        size = _d.pop(cls.PARAM_SIZE, None)
+        nfiles = _d.pop(cls.PARAM_NFILES, None)
+
+        if not _d:
+            return cls(None, None)
+
+        ((name, value),) = _d.items()
+        return cls(name, value, size=size, nfiles=nfiles)
+
+    def to_dict(self):
+        ret = OrderedDict()
+        if not self:
+            return ret
+
+        ret[self.name] = self.value
+        if self.size is not None:
+            ret[self.PARAM_SIZE] = self.size
+        if self.nfiles is not None:
+            ret[self.PARAM_NFILES] = self.nfiles
+        return ret
+
+    @property
+    def isdir(self):
+        if not self:
+            return False
+        return self.value.endswith(HASH_DIR_SUFFIX)

dvc/ignore.py

@@ -1,16 +1,17 @@
 import logging
 import os
 import re
+from collections import namedtuple
 from itertools import groupby, takewhile
 
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
-from pygtrie import StringTrie
 
 from dvc.path_info import PathInfo
-from dvc.pathspec_math import merge_patterns
+from dvc.pathspec_math import PatternInfo, merge_patterns
 from dvc.system import System
 from dvc.utils import relpath
+from dvc.utils.collections import PathStringTrie
 
 logger = logging.getLogger(__name__)
 
@@ -24,28 +25,44 @@ class DvcIgnore:
 
 class DvcIgnorePatterns(DvcIgnore):
     def __init__(self, pattern_list, dirname):
+        if pattern_list:
+            if isinstance(pattern_list[0], str):
+                pattern_list = [
+                    PatternInfo(pattern, "") for pattern in pattern_list
+                ]
 
         self.pattern_list = pattern_list
         self.dirname = dirname
         self.prefix = self.dirname + os.sep
 
-        regex_pattern_list = map(
-            GitWildMatchPattern.pattern_to_regex, pattern_list
-        )
+        self.regex_pattern_list = [
+            GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
+            for pattern_info in pattern_list
+        ]
 
         self.ignore_spec = [
             (ignore, re.compile("|".join(item[0] for item in group)))
-            for ignore, group in groupby(regex_pattern_list, lambda x: x[1])
+            for ignore, group in groupby(
+                self.regex_pattern_list, lambda x: x[1]
+            )
             if ignore is not None
         ]
 
     @classmethod
-    def from_files(cls, ignore_file_path, tree):
+    def from_files(cls, ignore_file_path, fs):
         assert os.path.isabs(ignore_file_path)
         dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-        with tree.open(ignore_file_path, encoding="utf-8") as fobj:
+        ignore_file_rel_path = os.path.relpath(ignore_file_path, fs.fs_root)
+        with fs.open(ignore_file_path, encoding="utf-8") as fobj:
             path_spec_lines = [
-                line for line in map(str.strip, fobj.readlines()) if line
+                PatternInfo(
+                    line,
+                    "{}:{}:{}".format(ignore_file_rel_path, line_no + 1, line),
+                )
+                for line_no, line in enumerate(
+                    map(str.strip, fobj.readlines())
+                )
+                if line and not (line.strip().startswith("#"))
             ]
 
         return cls(path_spec_lines, dirname)
@@ -56,7 +73,7 @@ class DvcIgnorePatterns(DvcIgnore):
 
         return dirs, files
 
-    def matches(self, dirname, basename, is_dir=False):
+    def _get_normalize_path(self, dirname, basename):
         # NOTE: `relpath` is too slow, so we have to assume that both
         # `dirname` and `self.dirname` are relative or absolute together.
         if dirname == self.dirname:
@@ -70,40 +87,83 @@ class DvcIgnorePatterns(DvcIgnore):
 
         if not System.is_unix():
             path = normalize_file(path)
+        return path
+
+    def matches(self, dirname, basename, is_dir=False, details: bool = False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
+
+        if details:
+            return self._ignore_details(path, is_dir)
         return self.ignore(path, is_dir)
 
     def ignore(self, path, is_dir):
+        def matches(pattern, path, is_dir) -> bool:
+            matches_ = bool(pattern.match(path))
+
+            if is_dir:
+                matches_ |= bool(pattern.match(f"{path}/"))
+
+            return matches_
+
         result = False
-        if is_dir:
-            path_dir = f"{path}/"
-            for ignore, pattern in self.ignore_spec:
-                if pattern.match(path) or pattern.match(path_dir):
-                    result = ignore
-        else:
-            for ignore, pattern in self.ignore_spec:
-                if pattern.match(path):
-                    result = ignore
+
+        for ignore, pattern in self.ignore_spec[::-1]:
+            if matches(pattern, path, is_dir):
+                result = ignore
+                break
+        return result
+
+    def _ignore_details(self, path, is_dir: bool):
+        result = []
+        for (regex, _), pattern_info in list(
+            zip(self.regex_pattern_list, self.pattern_list)
+        ):
+            # skip system pattern
+            if not pattern_info.file_info:
+                continue
+
+            regex = re.compile(regex)
+
+            matches = bool(regex.match(path))
+            if is_dir:
+                matches |= bool(regex.match(f"{path}/"))
+
+            if matches:
+                result.append(pattern_info.file_info)
+
         return result
 
     def __hash__(self):
-        return hash(self.dirname + ":" + "\n".join(self.pattern_list))
+        return hash(self.dirname + ":" + str(self.pattern_list))
 
     def __eq__(self, other):
         if not isinstance(other, DvcIgnorePatterns):
             return NotImplemented
         return (self.dirname == other.dirname) & (
-            self.pattern_list == other.pattern_list
+            [pattern.patterns for pattern in self.pattern_list]
+            == [pattern.patterns for pattern in other.pattern_list]
         )
 
     def __bool__(self):
         return bool(self.pattern_list)
 
 
+CheckIgnoreResult = namedtuple(
+    "CheckIgnoreResult", ["file", "match", "patterns"]
+)
+
+
+def _no_match(path):
+    return CheckIgnoreResult(path, False, ["::"])
+
+
 class DvcIgnoreFilterNoop:
-    def __init__(self, tree, root_dir):
+    def __init__(self, fs, root_dir):
         pass
 
-    def __call__(self, root, dirs, files):
+    def __call__(self, root, dirs, files, **kwargs):
         return dirs, files
 
     def is_ignored_dir(self, _):
@@ -112,40 +172,43 @@ class DvcIgnoreFilterNoop:
     def is_ignored_file(self, _):
         return False
 
+    def check_ignore(self, path):
+        return _no_match(path)
+
+    def is_ignored(self, _):
+        return False
+
 
 class DvcIgnoreFilter:
-    @staticmethod
-    def _is_dvc_repo(root, directory):
-        from dvc.repo import Repo
-
-        return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
-
-    def __init__(self, tree, root_dir):
+    def __init__(self, fs, root_dir):
         from dvc.repo import Repo
 
         default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
 
-        self.tree = tree
+        self.fs = fs
         self.root_dir = root_dir
-        self.ignores_trie_tree = StringTrie(separator=os.sep)
-        self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
+        self.ignores_trie_fs = PathStringTrie()
+        self.ignores_trie_fs[root_dir] = DvcIgnorePatterns(
             default_ignore_patterns, root_dir
         )
+        self._ignored_subrepos = PathStringTrie()
         self._update(self.root_dir)
 
     def _update(self, dirname):
-        old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
+        self._update_sub_repo(dirname)
+
+        old_pattern = self.ignores_trie_fs.longest_prefix(dirname).value
         matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
 
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if not matches and self.tree.exists(
+        if not matches and self.fs.exists(
             ignore_file_path, use_dvcignore=False
         ):
             new_pattern = DvcIgnorePatterns.from_files(
-                ignore_file_path, self.tree
+                ignore_file_path, self.fs
             )
             if old_pattern:
-                self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
+                self.ignores_trie_fs[dirname] = DvcIgnorePatterns(
                     *merge_patterns(
                         old_pattern.pattern_list,
                         old_pattern.dirname,
@@ -154,46 +217,56 @@ class DvcIgnoreFilter:
                     )
                 )
             else:
-                self.ignores_trie_tree[dirname] = new_pattern
+                self.ignores_trie_fs[dirname] = new_pattern
         elif old_pattern:
-            self.ignores_trie_tree[dirname] = old_pattern
+            self.ignores_trie_fs[dirname] = old_pattern
 
-        # NOTE: using `walk` + `break` because tree doesn't have `listdir()`
-        for root, dirs, _ in self.tree.walk(dirname, use_dvcignore=False):
-            self._update_sub_repo(root, dirs)
-            break
+    def _update_sub_repo(self, path):
+        from dvc.repo import Repo
 
-    def _update_sub_repo(self, root, dirs):
-        for d in dirs:
-            if self._is_dvc_repo(root, d):
-                old_pattern = self.ignores_trie_tree.longest_prefix(root).value
-                if old_pattern:
-                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
-                        *merge_patterns(
-                            old_pattern.pattern_list,
-                            old_pattern.dirname,
-                            ["/{}/".format(d)],
-                            root,
-                        )
-                    )
-                else:
-                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
-                        ["/{}/".format(d)], root
-                    )
+        if path == self.root_dir:
+            return
+
+        dvc_dir = os.path.join(path, Repo.DVC_DIR)
+        if not os.path.exists(dvc_dir):
+            return
+
+        root, dname = os.path.split(path)
+        self._ignored_subrepos[root] = self._ignored_subrepos.get(
+            root, set()
+        ) | {dname}
+        pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
+        new_pattern = DvcIgnorePatterns([pattern_info], root)
+        old_pattern = self.ignores_trie_fs.longest_prefix(root).value
+        if old_pattern:
+            self.ignores_trie_fs[root] = DvcIgnorePatterns(
+                *merge_patterns(
+                    old_pattern.pattern_list,
+                    old_pattern.dirname,
+                    new_pattern.pattern_list,
+                    new_pattern.dirname,
+                )
+            )
+        else:
+            self.ignores_trie_fs[root] = new_pattern
+
+    def __call__(self, root, dirs, files, ignore_subrepos=True):
+        for dname in dirs:
+            self._update_sub_repo(os.path.join(root, dname))
 
-    def __call__(self, root, dirs, files):
         ignore_pattern = self._get_trie_pattern(root)
         if ignore_pattern:
-            return ignore_pattern(root, dirs, files)
-        else:
-            return dirs, files
+            dirs, files = ignore_pattern(root, dirs, files)
+            if not ignore_subrepos:
+                dirs.extend(self._ignored_subrepos.get(root, []))
+        return dirs, files
 
     def _get_trie_pattern(self, dirname):
-        ignore_pattern = self.ignores_trie_tree.get(dirname)
+        ignore_pattern = self.ignores_trie_fs.get(dirname)
         if ignore_pattern:
             return ignore_pattern
 
-        prefix = self.ignores_trie_tree.longest_prefix(dirname).key
+        prefix = self.ignores_trie_fs.longest_prefix(dirname).key
         if not prefix:
             # outside of the repo
             return None
@@ -210,26 +283,32 @@ class DvcIgnoreFilter:
         for parent in dirs:
             self._update(parent)
 
-        return self.ignores_trie_tree.get(dirname)
+        return self.ignores_trie_fs.get(dirname)
 
     def _is_ignored(self, path, is_dir=False):
         if self._outside_repo(path):
-            return True
+            return False
         dirname, basename = os.path.split(os.path.normpath(path))
         ignore_pattern = self._get_trie_pattern(dirname)
         if ignore_pattern:
             return ignore_pattern.matches(dirname, basename, is_dir)
-        else:
-            return False
+        return False
 
-    def is_ignored_dir(self, path):
+    def _is_subrepo(self, path):
+        dirname, basename = os.path.split(os.path.normpath(path))
+        return basename in self._ignored_subrepos.get(dirname, set())
+
+    def is_ignored_dir(self, path, ignore_subrepos=True):
         path = os.path.abspath(path)
+        if not ignore_subrepos:
+            return not self._is_subrepo(path)
         if path == self.root_dir:
             return False
 
         return self._is_ignored(path, True)
 
     def is_ignored_file(self, path):
+        path = os.path.abspath(path)
         return self._is_ignored(path, False)
 
     def _outside_repo(self, path):
@@ -245,3 +324,43 @@ class DvcIgnoreFilter:
         ):
             return True
         return False
+
+    def check_ignore(self, target):
+        # NOTE: can only be used in `dvc check-ignore`, see
+        # https://github.com/iterative/dvc/issues/5046
+        full_target = os.path.abspath(target)
+        if not self._outside_repo(full_target):
+            dirname, basename = os.path.split(os.path.normpath(full_target))
+            pattern = self._get_trie_pattern(dirname)
+            if pattern:
+                matches = pattern.matches(
+                    dirname, basename, os.path.isdir(full_target), True,
+                )
+
+                if matches:
+                    return CheckIgnoreResult(target, True, matches)
+        return _no_match(target)
+
+    def is_ignored(self, path):
+        # NOTE: can't use self.check_ignore(path).match for now, see
+        # https://github.com/iterative/dvc/issues/4555
+        if os.path.isfile(path):
+            return self.is_ignored_file(path)
+        if os.path.isdir(path):
+            return self.is_ignored_dir(path)
+        return self.is_ignored_file(path) or self.is_ignored_dir(path)
+
+
+def init(path):
+    dvcignore = os.path.join(path, DvcIgnore.DVCIGNORE_FILE)
+    if os.path.exists(dvcignore):
+        return dvcignore
+
+    with open(dvcignore, "w") as fobj:
+        fobj.write(
+            "# Add patterns of files dvc should ignore, which could improve\n"
+            "# the performance. Learn more at\n"
+            "# https://dvc.org/doc/user-guide/dvcignore\n"
+        )
+
+    return dvcignore