dvc: bump to 0.94.1

* snap: drop sudo

Fixes #3989

* snap: ci: fix builds

* snap: ci: developer notifications

* snap: minimise size

* debug commit

* Revert "snap: minimise size"

This reverts commit 77b41a96f68b2c8df833803d0a644172b7db4779.

* fix image info

* Revert "debug commit"

This reverts commit 232baf0ad78335b0f3806456a3f20b82cdf0f24c.

* snap: v1 channels

Related: #3872

* snap: v0 channels

Related #3990
Related #3989

* snap: auto-version (major) detect

* ci: fetch --tags explanation & error checking

* snap: update warning framework

* hardocde warning

* add deprecation dev comment

* remove warning on explicit v0 channel

* tidy comment

* fix py2

.codeclimate.yml

@@ -0,0 +1,5 @@
+languages:
+    Python: true
+
+exclude_paths:
+    - tests/

.codecov.yml

@@ -1,10 +1 @@
-comment: false
-coverage:
-  status:
-    project:
-      default:
-        target: 85
-        threshold: 10%
-    patch:
-      default:
-        threshold: 25%
+

.coveragerc

@@ -1,14 +1,10 @@
 [run]
-branch = True
-source = dvc
 omit =
     *__main__.py
 
 [report]
 exclude_lines =
     if __name__ == .__main__.:
-    if typing.TYPE_CHECKING:
-    if TYPE_CHECKING:
     pass
     raise NotImplementedError
     # pragma: no cover

.deepsource.toml

@@ -0,0 +1,15 @@
+version = 1
+
+test_patterns = [
+  'tests/**'
+]
+
+exclude_patterns = [
+  'dvc/utils/compat.py'
+]
+
+[[analyzers]]
+name = 'python'
+enabled = true
+runtime_version = '3.x.x'
+

.dvc/.gitignore

@@ -9,4 +9,3 @@
 /pkg
 /repos
 /tmp
-/experiments

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+ignore =
+    E203, # Whitespace before ':'
+    E266, # Too many leading '#' for block comment
+    W503, # Line break occurred before a binary operator
+max-line-length = 79
+select = B,C,E,F,W,T4,B9

.github/FUNDING.yml

@@ -1,3 +0,0 @@
-# These are supported funding model platforms
-
-patreon: DVCorg

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,61 +1,10 @@
 ---
-name: "\U0001F41B Bug Report"
-about: Create a bug report to help us improve DVC
+name: Bug report
+about: Create a report to help us improve
+
 ---
 
-# Bug Report
 
-<!--
-## Issue name
+**Please provide information about your setup**
+DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
 
-Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug. 
-
-Example: `repro: doesn't detect input changes`
--->
-
-## Description
-
-<!--
-A clear and concise description of what the bug is.
--->
-
-### Reproduce
-
-<!--
-Step list of how to reproduce the bug
--->
-
-<!--
-Example:
-
-1. dvc init
-2. Copy dataset.zip to the directory
-3. dvc add dataset.zip
-4. dvc run -d dataset.zip -o model ./train.sh
-5. modify dataset.zip
-6. dvc repro
--->
-
-### Expected
-
-<!--
-A clear and concise description of what you expect to happen.
--->
-
-### Environment information
-
-<!--
-This is required to ensure that we can reproduce the bug.
--->
-
-**Output of `dvc version`:**
-
-```console
-$ dvc version
-```
-
-**Additional Information (if any):**
-
-<!--
-If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
--->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: "🤗 Need help?"
-    url: https://dvc.org/chat
-    about: If you have a question, ask us on Discord. Please join with this invite 👉

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,5 +1,6 @@
 ---
-name: "\U0001F680 Feature Request"
+name: Feature request
 about: Suggest an idea for this project
+
 ---
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,7 @@
 * [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
 
-* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).
+
+* [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)
 
 Thank you for the contribution - we'll try to review it as soon as possible. 🙏

.github/mergify.yml

@@ -1,9 +0,0 @@
-pull_request_rules:
-  - name: backport patches to 1.11
-    conditions:
-      - base=master
-      - label=backport-1.11
-    actions:
-      backport:
-        branches:
-          - "1.11"

.github/release-drafter.yml

@@ -1,49 +1,5 @@
 # Config for https://github.com/apps/release-drafter
-name-template: "$RESOLVED_VERSION 🦉 "
-tag-template: "$RESOLVED_VERSION"
-categories:
-  - title: "🚀 New Features and Enhancements "
-    labels:
-      - "feature"
-      - "enhancement"
-      - "ui"
-  - title: "💣 Breaking Changes "
-    labels:
-      - "breaking-change"
-  - title: "🏇 Optimizations "
-    labels:
-      - "optimize"
-      - "performance"
-  - title: "🐛 Bug Fixes "
-    labels:
-      - "bugfix"
-      - "bug"
-  - title: "🔨 Maintenance "
-    labels:
-      - "maintenance"
-      - "refactoring"
-      - "chore"
-      - "build"
-      - "ci"
-change-template: "- $TITLE (#$NUMBER) @$AUTHOR"
-version-resolver:
-  major:
-    labels:
-      - "major"
-  minor:
-    labels:
-      - "minor"
-  patch:
-    labels:
-      - "patch"
-  default: patch
-exclude-labels:
-  - "skip-changelog"
+branches:
+  - master
 template: |
-  > Refer to https://dvc.org/doc/install for installation instructions.
-
-  ## Changes
-
   $CHANGES
-
-  Thanks again to $CONTRIBUTORS for the contributions! 🎉

.github/workflows/packages.yaml

@@ -1,131 +0,0 @@
-name: Build packages
-on:
-  push:
-    branches:
-    - master
-  release:
-    types: [released, prereleased]
-jobs:
-  linux:
-    runs-on: ubuntu-16.04
-    steps:
-    - uses: actions/[email protected]
-    - name: Set up Python 3.7
-      uses: actions/[email protected]
-      with:
-        python-version: 3.7
-    - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
-    - name: Publish deb
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
-        asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
-        asset_content_type: binary/octet-stream
-    - name: Publish rpm
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_content_type: binary/octet-stream
-  mac:
-    runs-on: macos-10.15
-    steps:
-    - uses: actions/[email protected]
-    - name: Set up Python 3.7
-      uses: actions/[email protected]
-      with:
-        python-version: 3.7
-    - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
-    - name: Publish
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
-        asset_name: dvc-${{ github.event.release.tag_name }}.pkg
-        asset_content_type: binary/octet-stream
-  windows:
-    runs-on: windows-2019
-    steps:
-    - uses: actions/[email protected]
-    - name: Set up Python 3.7
-      uses: actions/[email protected]
-      with:
-        python-version: 3.7
-    - name: Build
-      run: scripts\build_windows.cmd
-      shell: cmd
-    - name: Publish
-      if: github.event_name == 'release'
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected]
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.exe
-        asset_name: dvc-${{ github.event.release.tag_name }}.exe
-        asset_content_type: binary/octet-stream
-  python:
-    runs-on: ubuntu-16.04
-    steps:
-    - uses: actions/[email protected]
-    - name: Set up Python 3.7
-      uses: actions/[email protected]
-      with:
-        python-version: 3.7
-    - name: Install
-      run: pip install ".[all,tests]"
-    - name: Build packages
-      run: ./scripts/build_package.sh
-    - name: Publish packages to PyPI
-      if: github.event_name == 'release'
-      uses: pypa/gh-action-pypi-publish@master
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_TOKEN }}
-  snap:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/[email protected]
-    - name: Build the package
-      uses: snapcore/action-build@v1
-      id: snapcraft
-#    - name: Test the package
-#      run: |
-#        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
-#        dvc version
-#        sudo snap remove dvc
-    - name: Publish to the edge channel
-      uses: snapcore/action-publish@v1
-      if: github.event_name == 'push'
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: edge,v2/edge
-    - name: Publish to the stable channel
-      uses: snapcore/action-publish@v1
-      if: ${{ github.event_name == 'release' && github.event.release.released }}
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: stable,v2/stable

.github/workflows/tests.yaml

@@ -1,107 +0,0 @@
-name: Tests
-
-on:
-  pull_request:
-    branches:
-      - master
-  push:
-    branches:
-      - master
-
-env:
-  DVC_TEST: "true"
-  HOMEBREW_NO_AUTO_UPDATE: 1
-  SHELL: /bin/bash
-  GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/[email protected]
-    - name: Set up Python 3.8
-      uses: actions/[email protected]
-      with:
-        python-version: 3.8
-    - name: Install requirements
-      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
-    - name: install temporary dependencies
-      run: |
-        pip install git+https://github.com/isidentical/gdrivefs@service-account
-        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
-    - name: Check README
-      run: python setup.py checkdocs
-    - uses: pre-commit/[email protected]
-  tests:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-18.04, windows-2019, macos-10.15]
-        pyv: ["3.6", "3.7", "3.8", "3.9"]
-# macos 11.0 workers take too long to start up
-#        include:
-#          - os: "macos-11.0"
-#            pyv: "3.9"
-        exclude:
-          - os: macos-10.15
-            pyv: "3.9"
-
-    steps:
-    - uses: actions/[email protected]
-    - name: Set up Python
-      uses: actions/[email protected]
-      with:
-        python-version: ${{ matrix.pyv }}
-    - name: get pip cache dir
-      id: pip-cache-dir
-      run: |
-        echo "::set-output name=dir::$(pip cache dir)"
-    - name: set pip cache
-      id: pip-cache
-      uses: actions/cache@v2
-      with:
-        path: ${{ steps.pip-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: install
-      if: steps.cache.pip-cache-dir.cache-hit != 'true'
-      run: |
-        pip install wheel
-        pip install -e ".[all,tests]"
-    - name: install temporary dependencies
-      run: |
-        pip install git+https://github.com/isidentical/gdrivefs@service-account
-        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
-    - name: setup git
-      run: |
-        git config --global user.email "[email protected]"
-        git config --global user.name "DVC Tester"
-    - name: setup gs creds
-      shell: bash
-      if: env.GS_CREDS_JSON != ''
-      run: |
-        mkdir -p scripts/ci
-        echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
-    - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
-    - name: upload coverage report
-      uses: codecov/[email protected]
-      with:
-        file: ./coverage.xml
-        fail_ci_if_error: false
-    - name: Post TAP
-      # Running custom fork with an option, that does not fail CI on error
-      uses: skshetry/flaky-service/packages/action@master
-      if: always() && github.ref == 'refs/heads/master'
-      with:
-        file-path: ${{github.workspace}}/testresults.tap
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        os: ${{runner.os}}
-        matrix: ${{toJson(matrix)}}
-        repo-description: ${{github.event.repository.description}}
-        fail_ci_if_error: false

.gitignore

@@ -1,12 +1,12 @@
 neatlynx.conf
 neatlynx/__pycache__
 .idea
-/cache
+cache
 *.pyc
 .env/
 .env2.7/
-.python-version
 
+cache
 .dvc.conf.lock
 .DS_Store
 build/

.pre-commit-config.yaml

@@ -1,77 +1,35 @@
 repos:
-  - hooks:
-      - id: black
-        language_version: python3
-    repo: https://github.com/ambv/black
-    rev: 19.10b0
-  - hooks:
-      - id: isort
-        language_version: python3
-    repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
-  - hooks:
-      - id: flake8
-        language_version: python3
-        additional_dependencies:
-          - flake8-bugbear
-          - flake8-comprehensions
-          - flake8-debugger
-          - flake8-string-format
-    repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
-  - hooks:
-      -  id: mypy
-         language_version: python3
-         files: ^dvc/
-    repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
-  - repo: local
-    hooks:
-    - id: pylint
-      name: pylint
-      entry: pylint
-      language: system
-      types: [python]
-  - hooks:
-      - args:
-          - -i
-          - "2"
-        id: beautysh
-        language_version: python3
-    repo: https://github.com/lovesegfault/beautysh
-    rev: 6.0.1
-  - hooks:
-      - args:
-          - git-hook
-          - pre-commit
-        entry: dvc
-        id: dvc-pre-commit
-        language: system
-        name: DVC pre-commit
-        stages:
-          - commit
-        verbose: true
-        require_serial: true
-      - args:
-          - git-hook
-          - pre-push
-        entry: dvc
-        id: dvc-pre-push
-        language: system
-        name: DVC pre-push
-        stages:
-          - push
-        require_serial: true
-      - always_run: true
-        args:
-          - git-hook
-          - post-checkout
-        entry: dvc
-        id: dvc-post-checkout
-        language: system
-        minimum_pre_commit_version: 2.2.0
-        name: DVC post-checkout
-        stages:
-          - post-checkout
-        require_serial: true
-    repo: local
+- hooks:
+  - id: black
+    language_version: python3
+  repo: https://github.com/ambv/black
+  rev: 19.10b0
+- hooks:
+  - id: flake8
+    language_version: python3
+  repo: https://gitlab.com/pycqa/flake8
+  rev: 3.7.9
+- hooks:
+  - args:
+    - -i
+    - '2'
+    id: beautysh
+    language_version: python3
+  repo: https://github.com/lovesegfault/beautysh
+  rev: master
+- hooks:
+  - id: dvc-pre-commit
+    language_version: python3
+    stages:
+    - commit
+  - id: dvc-pre-push
+    language_version: python3
+    stages:
+    - push
+  - always_run: true
+    id: dvc-post-checkout
+    language_version: python3
+    stages:
+    - post-checkout
+  repo: https://github.com/iterative/dvc
+  rev: master

.pre-commit-hooks.yaml

@@ -6,7 +6,6 @@
   language: python
   language_version: python3
   name: DVC pre-commit
-  require_serial: true
   stages:
   - commit
   verbose: true
@@ -18,7 +17,6 @@
   language: python
   language_version: python3
   name: DVC pre-push
-  require_serial: true
   stages:
   - push
 - always_run: true
@@ -31,6 +29,5 @@
   language_version: python3
   minimum_pre_commit_version: 2.2.0
   name: DVC post-checkout
-  require_serial: true
   stages:
   - post-checkout

.pylintrc

@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
-extension-pkg-whitelist=pygit2
+extension-pkg-whitelist=
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
@@ -15,11 +15,7 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook="import sys; \
-           from pathlib import Path; \
-           from pylint.config import find_pylintrc; \
-           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
-
+#init-hook=
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
@@ -32,7 +28,7 @@ limit-inference-results=100
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
-load-plugins=pylint_pytest,pylint_plugin_disable
+load-plugins=
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -64,16 +60,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use "--disable=all --enable=classes
 # --disable=W".
-disable=format,
-        # ^ most of them are covered by flake8,
-        # and are conventions rather than warnings or errors
-        refactoring,  # will be noisy if it starts giving suggestions
-        spelling,
-        # design is too-few-* and too-many-* checks, too-much-* for us right now
-        design,
-        # following were generated by default
-        # most of them are redundant Py3 checks for Py2+3 codebase
-        print-statement,
+disable=print-statement,
         parameter-unpacking,
         unpacking-in-except,
         old-raise-syntax,
@@ -152,40 +139,15 @@ disable=format,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        # custom disables start here
-        no-self-use,
-        invalid-name,
-        # already a check enabled for `wildcard-import`
-        unused-wildcard-import,
-        misplaced-comparison-constant,
-        # probably soon: https://github.com/iterative/dvc/issues/1843
-        logging-format-interpolation,
-        logging-fstring-interpolation,
-        # we don't have proper docstrings yet
-        # even if we do, we'll be doing it gradually
-        missing-function-docstring,
-        missing-module-docstring,
-        missing-class-docstring,
-        duplicate-code,
-        # too noisy, and detects issues even when import is inside a function
-        # see: https://github.com/PyCQA/pylint/issues/850
-        cyclic-import,
-        # isort should cover these:
-        wrong-import-order,
-        wrong-import-position,
-        ungrouped-imports,
-        multiple-imports,
-        # we use it to speedup/optimize
-        import-outside-toplevel,
-        fixme,
-
+        useless-object-inheritance,
+        missing-docstring,
+        bad-continuation
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once). See also the "--disable" option for examples.
-enable=c-extension-no-member,
-       no-else-return,
+enable=c-extension-no-member
 
 
 [REPORTS]
@@ -266,7 +228,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=hdfs.*,pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
+generated-members=
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -287,13 +249,13 @@ ignore-on-opaque-inference=yes
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
+ignored-classes=optparse.Values,thread._local,_thread._local
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=azure
+ignored-modules=
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.
@@ -328,7 +290,7 @@ dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore.
-ignored-argument-names=_.*|^ignored_|^unused_|args|kwargs
+ignored-argument-names=_.*|^ignored_|^unused_
 
 # Tells whether we should check for unused import in __init__ files.
 init-import=no

.restyled.yaml

@@ -0,0 +1,11 @@
+---
+restylers:
+  - name: black
+    image: restyled/restyler-black:v19.10b0
+    command:
+    - black
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python

.travis.yml

@@ -0,0 +1,173 @@
+cache:
+  directories:
+    - $HOME/.cache/pip
+    - $HOME/.pyenv
+dist: xenial
+env:
+  global:
+    - HOMEBREW_NO_AUTO_UPDATE=1
+    - PYENV_ROOT="$HOME/.pyenv"
+    - PATH="$PYENV_ROOT/bin:$PATH"
+    - PIP_CACHE_DIR="$HOME/.cache/pip"  # unify pip cache location for all platforms
+stages:
+  - check
+  - test
+  - build
+# Travis doesn't support testing python on Mac yet, so we need
+# to workaround it by installing it directly with brew.
+#
+# Note that we specify language as ruby-2.4.3 to help us satisfy
+# fpm dependencies, as travis runs el capitan with an old ruby
+# version, that causes fpm to crash.
+#
+# Each job will use the previous job's `stage` tag if not specified.
+jobs:
+  include:
+  # patch formatting checks
+  - name: "Check patch formatting"
+    stage: check
+    os: linux
+    language: python
+    python: 3.7
+    before_install:
+    install:
+    script: ./scripts/ci/check_patch.sh
+  # test jobs
+  - name: "3.7 on Windows"
+    stage: test
+    os: windows           # Windows 10.0.17134 N/A Build 17134
+    language: shell       # 'language: python' is an error on Travis CI Windows
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
+  - name: "3.8 on Windows"
+    os: windows           # Windows 10.0.17134 N/A Build 17134
+    language: shell       # 'language: python' is an error on Travis CI Windows
+    env:
+      - PYTHON_VERSION=3.8.0
+      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
+  - name: "3.5 on Linux"
+    language: python
+    python: 3.5
+  - name: "3.6 on Linux"
+    language: python
+    python: 3.6
+  - name: "3.7 on Linux"
+    language: python
+    python: 3.7
+  - name: "3.8 on Linux"
+    language: python
+    python: 3.8
+  - name: "3.7 on Mac"
+    os: osx
+    osx_image: xcode11.3
+  # build jobs
+  - name: "Mac pkg"
+    stage: build
+    os: osx
+    osx_image: xcode9.4
+    language: ruby
+    rvm: 2.4.3
+    install:
+    script: ./scripts/build_posix.sh
+  - name: "Windows pkg"
+    os: windows
+    language: shell
+    install:
+    script: ./scripts/build_windows.cmd
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
+  - name: "Linux pkgs"
+    language: python
+    python: 3.7
+    before_install:
+    install:
+    script: ./scripts/build_posix.sh
+  - name: "PyPI pkgs"
+    language: python
+    python: 3.8
+    script: ./scripts/build_package.sh
+  - name: Snapcraft snap
+    language: python
+    python: 3.8
+    addons:
+     snaps:
+     - name: snapcraft
+       channel: stable
+       confinement: classic
+     - name: review-tools
+       channel: stable
+     - name: lxd
+       channel: stable
+    env:
+    - SNAPCRAFT_IMAGE_INFO: |
+        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
+    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
+    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
+    install:
+    - sudo usermod --append --groups lxd $USER
+    - sudo /snap/bin/lxd.migrate -yes
+    - sudo /snap/bin/lxd waitready
+    - sudo /snap/bin/lxd init --auto
+    script:
+    - ./scripts/build_snap.sh
+    after_failure:
+    - sudo journalctl -u snapd
+
+before_install:
+  - bash ./scripts/ci/before_install.sh
+  - source env.sh
+  - bash ./scripts/ci/check_python.sh
+install:
+  - bash ./scripts/ci/install.sh
+script:
+  - bash ./scripts/ci/script.sh
+deploy:
+  - provider: releases
+    api_key:
+      secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
+    file_glob: true
+    file:
+      - dvc*.rpm
+      - dvc*.deb
+      - dvc*.pkg
+      - dvc*.exe
+    skip_cleanup: true
+    on:
+      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
+      tags: true
+      repo: iterative/dvc
+      stage: build
+  - provider: pypi
+    user:
+      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
+    password:
+      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
+    skip_existing: true
+    file_glob: true
+    file:
+      - dist/dvc*.whl
+      - dist/dvc*.tar.gz
+    skip_cleanup: true
+    on:
+      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
+      tags: true
+      repo: iterative/dvc
+      stage: build
+  - provider: script
+    skip_cleanup: true
+    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
+    on:
+      all_branches: true
+      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
+      repo: iterative/dvc
+      stage: build
+  - provider: script
+    skip_cleanup: true
+    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push nowarn_update_dvc_*.snap --release $SNAP_CHANNEL_MAJOR || echo timed out"
+    on:
+      all_branches: true
+      condition: "$(./scripts/ci/deploy_condition.sh nowarn_update_dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL_MAJOR"
+      repo: iterative/dvc
+      stage: build

README.rst

@@ -10,7 +10,7 @@
 
 |Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
 
-|PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
+|Snap| |Choco| |Brew| |Conda| |PyPI| |Packages|
 
 |
 
@@ -101,8 +101,7 @@ Snap (Snapcraft/Linux)
    snap install dvc --classic
 
 This corresponds to the latest tagged release.
-Add ``--beta`` for the latest tagged release candidate,
-or ``--edge`` for the latest ``master`` version.
+Add ``--edge`` for the latest ``master`` version.
 
 Choco (Chocolatey/Windows)
 --------------------------
@@ -131,6 +130,8 @@ Conda (Anaconda)
 
    conda install -c conda-forge dvc
 
+Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
+
 pip (PyPI)
 ----------
 
@@ -201,6 +202,38 @@ Contributing
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+   :alt: 0
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+   :alt: 1
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+   :alt: 2
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+   :alt: 3
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+   :alt: 4
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+   :alt: 5
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+   :alt: 6
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+   :alt: 7
+
 Mailing List
 ============
 
@@ -230,9 +263,9 @@ Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
    :target: https://travis-ci.com/iterative/dvc/branches
    :alt: Release
 
-.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
-   :target: https://github.com/iterative/dvc/actions
-   :alt: GHA Tests
+.. |CI| image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev&logo=travis
+   :target: https://travis-ci.com/iterative/dvc/builds
+   :alt: Travis dev branch
 
 .. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
    :target: https://codeclimate.com/github/iterative/dvc

dvc/__init__.py

@@ -6,4 +6,5 @@ Make your data science projects reproducible and shareable.
 import dvc.logger
 from dvc.version import __version__  # noqa: F401
 
+
 dvc.logger.setup()

dvc/_debug.py

@@ -1,91 +0,0 @@
-from contextlib import ExitStack, contextmanager
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from argparse import Namespace
-
-
-@contextmanager
-def instrument(html_output=False):
-    """Run a statistical profiler"""
-    try:
-        from pyinstrument import Profiler  # pylint: disable=import-error
-    except ImportError:
-        print("Failed to run profiler, pyinstrument is not installed")
-        yield
-        return
-
-    profiler = Profiler()
-
-    profiler.start()
-    yield
-    profiler.stop()
-
-    if html_output:
-        profiler.open_in_browser()
-        return
-    print(profiler.output_text(unicode=True, color=True))
-
-
-@contextmanager
-def profile(dump_path: str = None):
-    """Run a cprofile"""
-    import cProfile
-
-    prof = cProfile.Profile()
-    prof.enable()
-
-    yield
-
-    prof.disable()
-    if not dump_path:
-        prof.print_stats(sort="cumtime")
-        return
-    prof.dump_stats(dump_path)
-
-
-@contextmanager
-def debug():
-    try:
-        yield
-    except Exception:  # pylint: disable=broad-except
-        try:
-            import ipdb as pdb  # noqa: T100, pylint: disable=import-error
-        except ImportError:
-            import pdb  # noqa: T100
-        pdb.post_mortem()
-
-        raise  # prevent from jumping ahead
-
-
-@contextmanager
-def debugtools(args: "Namespace" = None, **kwargs):
-    kw = vars(args) if args else {}
-    kw.update(kwargs)
-
-    with ExitStack() as stack:
-        if kw.get("pdb"):
-            stack.enter_context(debug())
-        if kw.get("cprofile") or kw.get("cprofile_dump"):
-            stack.enter_context(profile(kw.get("cprofile_dump")))
-        if kw.get("instrument") or kw.get("instrument_open"):
-            stack.enter_context(instrument(kw.get("instrument_open", False)))
-        yield
-
-
-def add_debugging_flags(parser):
-    from argparse import SUPPRESS
-
-    parser.add_argument(
-        "--cprofile", action="store_true", default=False, help=SUPPRESS,
-    )
-    parser.add_argument("--cprofile-dump", help=SUPPRESS)
-    parser.add_argument(
-        "--pdb", action="store_true", default=False, help=SUPPRESS,
-    )
-    parser.add_argument(
-        "--instrument", action="store_true", default=False, help=SUPPRESS,
-    )
-    parser.add_argument(
-        "--instrument-open", action="store_true", default=False, help=SUPPRESS,
-    )

dvc/analytics.py

@@ -1,6 +1,25 @@
 import json
 import logging
 import os
+import platform
+import requests
+import sys
+import tempfile
+import uuid
+
+import distro
+
+from dvc import __version__
+from dvc.config import Config, to_bool
+from dvc.daemon import daemon
+from dvc.exceptions import NotDvcRepoError
+from dvc.lock import Lock, LockError
+from dvc.repo import Repo
+from dvc.scm import SCM, NoSCM
+from dvc.scm.base import SCMError
+from dvc.utils import env2bool, is_binary
+from dvc.utils.fs import makedirs
+
 
 logger = logging.getLogger(__name__)
 
@@ -17,10 +36,6 @@ def collect_and_send_report(args=None, return_code=None):
     report as a JSON, where the _collector_ generates it and the _sender_
     removes it after sending it.
     """
-    import tempfile
-
-    from dvc.daemon import daemon
-
     report = {}
 
     # Include command execution information on the report only when available.
@@ -36,9 +51,6 @@ def collect_and_send_report(args=None, return_code=None):
 
 
 def is_enabled():
-    from dvc.config import Config, to_bool
-    from dvc.utils import env2bool
-
     if env2bool("DVC_TEST"):
         return False
 
@@ -58,12 +70,10 @@ def send(path):
     `collect_and_send_report`. Sending happens on another process,
     thus, the need of removing such file afterwards.
     """
-    import requests
-
     url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
-    with open(path) as fobj:
+    with open(path, "r") as fobj:
         report = json.load(fobj)
 
     report.update(_runtime_info())
@@ -77,11 +87,6 @@ def send(path):
 
 
 def _scm_in_use():
-    from dvc.exceptions import NotDvcRepoError
-    from dvc.repo import Repo
-    from dvc.scm import SCM, NoSCM
-    from dvc.scm.base import SCMError
-
     try:
         scm = SCM(root_dir=Repo.find_root())
         return type(scm).__name__
@@ -95,9 +100,6 @@ def _runtime_info():
     """
     Gather information from the environment where DVC runs to fill a report.
     """
-    from dvc import __version__
-    from dvc.utils import is_binary
-
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
@@ -108,11 +110,6 @@ def _runtime_info():
 
 
 def _system_info():
-    import platform
-    import sys
-
-    import distro
-
     system = platform.system()
 
     if system == "Windows":
@@ -151,12 +148,6 @@ def _find_or_create_user_id():
 
     IDs are generated randomly with UUID.
     """
-    import uuid
-
-    from dvc.config import Config
-    from dvc.lock import Lock, LockError
-    from dvc.utils.fs import makedirs
-
     config_dir = Config.get_dir("global")
     fname = os.path.join(config_dir, "user_id")
     lockfile = os.path.join(config_dir, "user_id.lock")
@@ -168,7 +159,7 @@ def _find_or_create_user_id():
     try:
         with Lock(lockfile):
             try:
-                with open(fname) as fobj:
+                with open(fname, "r") as fobj:
                     user_id = json.load(fobj)["user_id"]
 
             except (FileNotFoundError, ValueError, KeyError):
@@ -180,4 +171,6 @@ def _find_or_create_user_id():
             return user_id
 
     except LockError:
-        logger.debug(f"Failed to acquire '{lockfile}'")
+        logger.debug(
+            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
+        )

dvc/api.py

@@ -1,12 +1,16 @@
 import os
-from contextlib import _GeneratorContextManager as GCM
+from contextlib import contextmanager, _GeneratorContextManager as GCM
 
-from funcy import reraise
-
-from dvc.exceptions import OutputNotFoundError, PathMissingError
-from dvc.objects.stage import get_hash
-from dvc.path_info import PathInfo
 from dvc.repo import Repo
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.external_repo import external_repo
+
+
+class UrlNotDvcRepoError(DvcException):
+    """Thrown if the given URL is not a DVC repository."""
+
+    def __init__(self, url):
+        super().__init__("'{}' is not a DVC repository.".format(url))
 
 
 def get_url(path, repo=None, rev=None, remote=None):
@@ -15,27 +19,20 @@ def get_url(path, repo=None, rev=None, remote=None):
     in a DVC repo. For Git repos, HEAD is used unless a rev argument is
     supplied. The default remote is tried unless a remote argument is supplied.
 
-    Raises OutputNotFoundError if the file is not tracked by DVC.
+    Raises UrlNotDvcRepoError if repo is not a DVC project.
 
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
     """
-    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        path_info = PathInfo(_repo.root_dir) / path
-        with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            metadata = _repo.repo_fs.metadata(path_info)
-
-        if not metadata.is_dvc:
-            raise OutputNotFoundError(path, repo)
-
-        cloud = metadata.repo.cloud
-        hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb)
-        return cloud.get_url_for(remote, checksum=hash_info.value)
+    with _make_repo(repo, rev=rev) as _repo:
+        if not isinstance(_repo, Repo):
+            raise UrlNotDvcRepoError(_repo.url)
+        out = _repo.find_out_by_relpath(path)
+        remote_obj = _repo.cloud.get_remote(remote)
+        return str(remote_obj.checksum_to_path_info(out.checksum))
 
 
-def open(  # noqa, pylint: disable=redefined-builtin
-    path, repo=None, rev=None, remote=None, mode="r", encoding=None
-):
+def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
     """
     Open file in the supplied path tracked in a repo (both DVC projects and
     plain Git repos are supported). For Git repos, HEAD is used unless a rev
@@ -60,9 +57,7 @@ def open(  # noqa, pylint: disable=redefined-builtin
 
 
 class _OpenContextManager(GCM):
-    def __init__(
-        self, func, args, kwds
-    ):  # pylint: disable=super-init-not-called
+    def __init__(self, func, args, kwds):
         self.gen = func(*args, **kwds)
         self.func, self.args, self.kwds = func, args, kwds
 
@@ -73,7 +68,7 @@ class _OpenContextManager(GCM):
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
+    with _make_repo(repo, rev=rev) as _repo:
         with _repo.open_by_relpath(
             path, remote=remote, mode=mode, encoding=encoding
         ) as fd:
@@ -92,33 +87,14 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
         return fd.read()
 
 
-def make_checkpoint():
-    """
-    Signal DVC to create a checkpoint experiment.
-
-    If the current process is being run from DVC, this function will block
-    until DVC has finished creating the checkpoint. Otherwise, this function
-    will return immediately.
-    """
-    import builtins
-    from time import sleep
-
-    from dvc.env import DVC_CHECKPOINT, DVC_ROOT
-    from dvc.stage.monitor import CheckpointTask
-
-    if os.getenv(DVC_CHECKPOINT) is None:
-        return
-
-    root_dir = os.getenv(DVC_ROOT, Repo.find_root())
-    signal_file = os.path.join(
-        root_dir, Repo.DVC_DIR, "tmp", CheckpointTask.SIGNAL_FILE
-    )
-
-    with builtins.open(signal_file, "w") as fobj:
-        # NOTE: force flushing/writing empty file to disk, otherwise when
-        # run in certain contexts (pytest) file may not actually be written
-        fobj.write("")
-        fobj.flush()
-        os.fsync(fobj.fileno())
-    while os.path.exists(signal_file):
-        sleep(0.1)
+@contextmanager
+def _make_repo(repo_url=None, rev=None):
+    repo_url = repo_url or os.getcwd()
+    if rev is None and os.path.exists(repo_url):
+        try:
+            yield Repo(repo_url)
+            return
+        except NotDvcRepoError:
+            pass  # fallthrough to external_repo
+    with external_repo(url=repo_url, rev=rev) as repo:
+        yield repo

dvc/cache.py

@@ -1,85 +1,74 @@
+"""Manages cache of a DVC repo."""
 from collections import defaultdict
 
-from dvc.scheme import Schemes
+from funcy import cached_property
 
 
-def get_odb(fs):
-    from .base import ObjectDB
-    from .gdrive import GDriveObjectDB
-    from .local import LocalObjectDB
-    from .ssh import SSHObjectDB
+def _make_remote_property(name):
+    """
+    The config file is stored in a way that allows you to have a
+    cache for each remote.
 
-    if fs.scheme == Schemes.LOCAL:
-        return LocalObjectDB(fs)
+    This is needed when specifying external outputs
+    (as they require you to have an external cache location).
 
-    if fs.scheme == Schemes.SSH:
-        return SSHObjectDB(fs)
+    Imagine a config file like the following:
 
-    if fs.scheme == Schemes.GDRIVE:
-        return GDriveObjectDB(fs)
+            ['remote "dvc-storage"']
+            url = ssh://localhost/tmp
+            ask_password = true
 
-    return ObjectDB(fs)
+            [cache]
+            ssh = dvc-storage
+
+    This method creates a cached property, containing cache named `name`:
+
+        self.config == {'ssh': 'dvc-storage'}
+        self.ssh  # a RemoteSSH instance
+    """
+
+    def getter(self):
+        from dvc.remote import Remote
+
+        remote = self.config.get(name)
+        if not remote:
+            return None
+
+        return Remote(self.repo, name=remote)
+
+    getter.__name__ = name
+    return cached_property(getter)
 
 
-def _get_odb(repo, settings):
-    from dvc.fs import get_cloud_fs
+class Cache(object):
+    """Class that manages cache locations of a DVC repo.
 
-    if not settings:
-        return None
+    Args:
+        repo (dvc.repo.Repo): repo instance that this cache belongs to.
+    """
 
-    fs = get_cloud_fs(repo, **settings)
-    return get_odb(fs)
-
-
-class ODBManager:
     CACHE_DIR = "cache"
-    CLOUD_SCHEMES = [
-        Schemes.S3,
-        Schemes.GS,
-        Schemes.SSH,
-        Schemes.HDFS,
-        Schemes.WEBHDFS,
-    ]
 
     def __init__(self, repo):
+        from dvc.remote import Remote
+
         self.repo = repo
         self.config = config = repo.config["cache"]
-        self._odb = {}
 
         local = config.get("local")
 
         if local:
             settings = {"name": local}
-        elif "dir" not in config:
-            settings = None
         else:
-            from dvc.config_schema import LOCAL_COMMON
+            settings = {**config, "url": config["dir"]}
 
-            settings = {"url": config["dir"]}
-            for opt in LOCAL_COMMON.keys():
-                if opt in config:
-                    settings[str(opt)] = config.get(opt)
+        self.local = Remote(repo, **settings)
 
-        self._odb[Schemes.LOCAL] = _get_odb(repo, settings)
-
-    def _init_odb(self, schemes):
-        for scheme in schemes:
-            remote = self.config.get(scheme)
-            settings = {"name": remote} if remote else None
-            self._odb[scheme] = _get_odb(self.repo, settings)
-
-    def __getattr__(self, name):
-        if name not in self._odb and name in self.CLOUD_SCHEMES:
-            self._init_odb([name])
-
-        try:
-            return self._odb[name]
-        except KeyError as exc:
-            raise AttributeError from exc
-
-    def by_scheme(self):
-        self._init_odb(self.CLOUD_SCHEMES)
-        yield from self._odb.items()
+    s3 = _make_remote_property("s3")
+    gs = _make_remote_property("gs")
+    ssh = _make_remote_property("ssh")
+    hdfs = _make_remote_property("hdfs")
+    azure = _make_remote_property("azure")
 
 
 class NamedCacheItem:
@@ -112,8 +101,7 @@ class NamedCacheItem:
             self.children[checksum].update(child_item)
 
 
-class NamedCache:
-    # pylint: disable=protected-access
+class NamedCache(object):
     def __init__(self):
         self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
         self.external = defaultdict(set)

dvc/checkout.py

@@ -1,302 +0,0 @@
-import logging
-
-from shortuuid import uuid
-
-import dvc.prompt as prompt
-from dvc.exceptions import (
-    CacheLinkError,
-    CheckoutError,
-    ConfirmRemoveError,
-    DvcException,
-)
-from dvc.objects import ObjectFormatError, check, load
-from dvc.objects.stage import get_hash
-from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
-    slow_link_guard,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def _changed(path_info, fs, obj, cache):
-    logger.trace("checking if '%s'('%s') has changed.", path_info, obj)
-
-    try:
-        check(cache, obj)
-    except (FileNotFoundError, ObjectFormatError):
-        logger.debug(
-            "cache for '%s'('%s') has changed.", path_info, obj.hash_info
-        )
-        return True
-
-    try:
-        actual = get_hash(path_info, fs, obj.hash_info.name, cache)
-    except FileNotFoundError:
-        logger.debug("'%s' doesn't exist.", path_info)
-        return True
-
-    if obj.hash_info != actual:
-        logger.debug(
-            "hash value '%s' for '%s' has changed (actual '%s').",
-            obj.hash_info,
-            actual,
-            path_info,
-        )
-        return True
-
-    logger.trace("'%s' hasn't changed.", path_info)
-    return False
-
-
-def _remove(path_info, fs, cache, force=False):
-    if not fs.exists(path_info):
-        return
-
-    if force:
-        fs.remove(path_info)
-        return
-
-    current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache)
-    try:
-        obj = load(cache, current)
-        check(cache, obj)
-    except (FileNotFoundError, ObjectFormatError):
-        msg = (
-            f"file/directory '{path_info}' is going to be removed. "
-            "Are you sure you want to proceed?"
-        )
-
-        if not prompt.confirm(msg):
-            raise ConfirmRemoveError(str(path_info))
-
-    fs.remove(path_info)
-
-
-def _verify_link(cache, path_info, link_type):
-    if link_type == "hardlink" and cache.fs.getsize(path_info) == 0:
-        return
-
-    if cache.cache_type_confirmed:
-        return
-
-    is_link = getattr(cache.fs, f"is_{link_type}", None)
-    if is_link and not is_link(path_info):
-        cache.fs.remove(path_info)
-        raise DvcException(f"failed to verify {link_type}")
-
-    cache.cache_type_confirmed = True
-
-
-def _do_link(cache, from_info, to_info, link_method):
-    if cache.fs.exists(to_info):
-        raise DvcException(f"Link '{to_info}' already exists!")
-
-    link_method(from_info, to_info)
-
-    logger.debug(
-        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info,
-    )
-
-
-@slow_link_guard
-def _try_links(cache, from_info, to_info, link_types):
-    while link_types:
-        link_method = getattr(cache.fs, link_types[0])
-        try:
-            _do_link(cache, from_info, to_info, link_method)
-            _verify_link(cache, to_info, link_types[0])
-            return
-
-        except DvcException as exc:
-            logger.debug(
-                "Cache type '%s' is not supported: %s", link_types[0], exc
-            )
-            del link_types[0]
-
-    raise CacheLinkError([to_info])
-
-
-def _link(cache, from_info, to_info):
-    assert cache.fs.isfile(from_info)
-    cache.makedirs(to_info.parent)
-    _try_links(cache, from_info, to_info, cache.cache_types)
-
-
-def _cache_is_copy(cache, path_info):
-    """Checks whether cache uses copies."""
-    if cache.cache_type_confirmed:
-        return cache.cache_types[0] == "copy"
-
-    if set(cache.cache_types) <= {"copy"}:
-        return True
-
-    workspace_file = path_info.with_name("." + uuid())
-    test_cache_file = cache.fs.path_info / ".cache_type_test_file"
-    if not cache.fs.exists(test_cache_file):
-        cache.makedirs(test_cache_file.parent)
-        with cache.fs.open(test_cache_file, "wb") as fobj:
-            fobj.write(bytes(1))
-    try:
-        _link(cache, test_cache_file, workspace_file)
-    finally:
-        cache.fs.remove(workspace_file)
-        cache.fs.remove(test_cache_file)
-
-    cache.cache_type_confirmed = True
-    return cache.cache_types[0] == "copy"
-
-
-def _checkout_file(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
-):
-    """The file is changed we need to checkout a new copy"""
-    modified = False
-    cache_info = cache.hash_to_path_info(obj.hash_info.value)
-    if fs.exists(path_info):
-        if not relink and _changed(path_info, fs, obj, cache):
-            modified = True
-            _remove(path_info, fs, cache, force=force)
-            _link(cache, cache_info, path_info)
-        else:
-            if fs.iscopy(path_info) and _cache_is_copy(cache, path_info):
-                cache.unprotect(path_info)
-            else:
-                _remove(path_info, fs, cache, force=force)
-                _link(cache, cache_info, path_info)
-    else:
-        _link(cache, cache_info, path_info)
-        modified = True
-
-    fs.repo.state.save(path_info, fs, obj.hash_info)
-    if progress_callback:
-        progress_callback(str(path_info))
-
-    return modified
-
-
-def _remove_redundant_files(path_info, fs, dir_info, cache, force):
-    existing_files = set(fs.walk_files(path_info))
-
-    needed_files = {path_info.joinpath(*key) for key, _ in dir_info.items()}
-    redundant_files = existing_files - needed_files
-    for path in redundant_files:
-        _remove(path, fs, cache, force)
-
-    return bool(redundant_files)
-
-
-def _checkout_dir(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
-):
-    modified = False, False
-    # Create dir separately so that dir is created
-    # even if there are no files in it
-    if not fs.exists(path_info):
-        modified = True
-        fs.makedirs(path_info)
-
-    logger.debug("Linking directory '%s'.", path_info)
-
-    for entry_key, entry_hash_info in obj.hash_info.dir_info.items():
-        entry_modified = _checkout_file(
-            path_info.joinpath(*entry_key),
-            fs,
-            cache.get(entry_hash_info),
-            cache,
-            force,
-            progress_callback,
-            relink,
-        )
-        if entry_modified:
-            modified = True
-
-    modified = (
-        _remove_redundant_files(
-            path_info, fs, obj.hash_info.dir_info, cache, force
-        )
-        or modified
-    )
-
-    fs.repo.state.save(path_info, fs, obj.hash_info)
-
-    # relink is not modified, assume it as nochange
-    return modified and not relink
-
-
-def _checkout(
-    path_info,
-    fs,
-    obj,
-    cache,
-    force=False,
-    progress_callback=None,
-    relink=False,
-):
-    if not obj.hash_info.isdir:
-        ret = _checkout_file(
-            path_info, fs, obj, cache, force, progress_callback, relink
-        )
-    else:
-        ret = _checkout_dir(
-            path_info, fs, obj, cache, force, progress_callback, relink,
-        )
-
-    fs.repo.state.save_link(path_info, fs)
-
-    return ret
-
-
-def checkout(
-    path_info,
-    fs,
-    obj,
-    cache,
-    force=False,
-    progress_callback=None,
-    relink=False,
-    quiet=False,
-):
-    if path_info.scheme not in ["local", cache.fs.scheme]:
-        raise NotImplementedError
-
-    failed = None
-    skip = False
-    if not obj:
-        if not quiet:
-            logger.warning(
-                "No file hash info found for '%s'. It won't be created.",
-                path_info,
-            )
-        _remove(path_info, fs, cache, force=force)
-        failed = path_info
-
-    elif not relink and not _changed(path_info, fs, obj, cache):
-        logger.trace("Data '%s' didn't change.", path_info)
-        skip = True
-    else:
-        try:
-            check(cache, obj)
-        except (FileNotFoundError, ObjectFormatError):
-            if not quiet:
-                logger.warning(
-                    "Cache '%s' not found. File '%s' won't be created.",
-                    obj.hash_info,
-                    path_info,
-                )
-            _remove(path_info, fs, cache, force=force)
-            failed = path_info
-
-    if failed or skip:
-        if progress_callback and obj:
-            progress_callback(
-                str(path_info), len(obj),
-            )
-        if failed:
-            raise CheckoutError([failed])
-        return
-
-    logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
-
-    return _checkout(
-        path_info, fs, obj, cache, force, progress_callback, relink,
-    )

dvc/cli.py

@@ -1,52 +1,45 @@
 """DVC command line interface"""
 import argparse
 import logging
-import os
 import sys
 
-from ._debug import add_debugging_flags
 from .command import (
     add,
     cache,
-    check_ignore,
     checkout,
     commit,
-    completion,
     config,
     daemon,
-    dag,
     data_sync,
     destroy,
     diff,
-    experiments,
-    freeze,
     gc,
     get,
     get_url,
-    git_hook,
     imp,
     imp_url,
     init,
     install,
-    live,
     ls,
+    lock,
     metrics,
-    move,
     params,
-    plots,
+    move,
+    pipeline,
     remote,
     remove,
     repro,
     root,
     run,
-    stage,
     unprotect,
     update,
     version,
+    git_hook,
 )
 from .command.base import fix_subparsers
 from .exceptions import DvcParserError
 
+
 logger = logging.getLogger(__name__)
 
 COMMANDS = [
@@ -73,54 +66,31 @@ COMMANDS = [
     install,
     root,
     ls,
-    freeze,
-    dag,
+    lock,
+    pipeline,
     daemon,
     commit,
-    completion,
     diff,
     version,
     update,
     git_hook,
-    plots,
-    stage,
-    experiments,
-    check_ignore,
-    live,
 ]
 
 
-def _find_parser(parser, cmd_cls):
-    defaults = parser._defaults  # pylint: disable=protected-access
-    if not cmd_cls or cmd_cls == defaults.get("func"):
-        parser.print_help()
-        raise DvcParserError()
-
-    actions = parser._actions  # pylint: disable=protected-access
-    for action in actions:
-        if not isinstance(action.choices, dict):
-            # NOTE: we are only interested in subparsers
-            continue
-        for subparser in action.choices.values():
-            _find_parser(subparser, cmd_cls)
-
-
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
 
-    def error(self, message, cmd_cls=None):  # pylint: disable=arguments-differ
-        logger.error(message)
-        _find_parser(self, cmd_cls)
+    def error(self, message):
+        """Custom error method.
+        Args:
+            message (str): error message.
 
-    def parse_args(self, args=None, namespace=None):
-        # NOTE: overriding to provide a more granular help message.
-        # E.g. `dvc plots diff --bad-flag` would result in a `dvc plots diff`
-        # help message instead of generic `dvc` usage.
-        args, argv = self.parse_known_args(args, namespace)
-        if argv:
-            msg = "unrecognized arguments: %s"
-            self.error(msg % " ".join(argv), getattr(args, "func", None))
-        return args
+        Raises:
+            dvc.exceptions.DvcParser: dvc parser exception.
+        """
+        logger.error(message)
+        self.print_help()
+        raise DvcParserError()
 
 
 class VersionAction(argparse.Action):  # pragma: no cover
@@ -142,19 +112,31 @@ def get_parent_parser():
     in order to prevent some weird behavior.
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
-    add_debugging_flags(parent_parser)
+
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
-        "-q", "--quiet", action="count", default=0, help="Be quiet."
+        "-q", "--quiet", action="store_true", default=False, help="Be quiet."
     )
     log_level_group.add_argument(
-        "-v", "--verbose", action="count", default=0, help="Be verbose."
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="Be verbose.",
     )
 
     return parent_parser
 
 
-def get_main_parser():
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
     parent_parser = get_parent_parser()
 
     # Main parser
@@ -189,14 +171,6 @@ def get_main_parser():
         help="Show program's version.",
     )
 
-    parser.add_argument(
-        "--cd",
-        default=os.path.curdir,
-        metavar="<path>",
-        help="Change to directory before executing.",
-        type=str,
-    )
-
     # Sub commands
     subparsers = parser.add_subparsers(
         title="Available Commands",
@@ -210,18 +184,6 @@ def get_main_parser():
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
-    return parser
-
-
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        dvc.exceptions.DvcParserError: raised for argument parsing errors.
-    """
-    parser = get_main_parser()
     args = parser.parse_args(argv)
+
     return args

dvc/command/add.py

@@ -1,19 +1,14 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link, CmdBase
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
 
 
 class CmdAdd(CmdBase):
     def run(self):
-        from dvc.exceptions import (
-            DvcException,
-            RecursiveAddingWhileUsingFilename,
-        )
-
         try:
             if len(self.args.targets) > 1 and self.args.file:
                 raise RecursiveAddingWhileUsingFilename()
@@ -23,13 +18,6 @@ class CmdAdd(CmdBase):
                 recursive=self.args.recursive,
                 no_commit=self.args.no_commit,
                 fname=self.args.file,
-                external=self.args.external,
-                glob=self.args.glob,
-                desc=self.args.desc,
-                out=self.args.out,
-                remote=self.args.remote,
-                to_remote=self.args.to_remote,
-                jobs=self.args.jobs,
             )
 
         except DvcException:
@@ -62,61 +50,12 @@ def add_parser(subparsers, parent_parser):
         help="Don't put files/directories into cache.",
     )
     parser.add_argument(
-        "--external",
-        action="store_true",
-        default=False,
-        help="Allow targets that are outside of the DVC repository.",
-    )
-    parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Allows targets containing shell-style wildcards.",
-    )
-    parser.add_argument(
+        "-f",
         "--file",
-        help="Specify name of the .dvc file this command will generate.",
+        help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     )
     parser.add_argument(
-        "-o",
-        "--out",
-        help="Destination path to put files to.",
-        metavar="<path>",
+        "targets", nargs="+", help="Input files/directories to add."
     )
-    parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Download it directly to the remote",
-    )
-    parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to download to",
-        metavar="<name>",
-    )
-    parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
-    parser.add_argument(
-        "--desc",
-        type=str,
-        metavar="<text>",
-        help=(
-            "User description of the data (optional). "
-            "This doesn't affect any DVC operations."
-        ),
-    )
-    parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add.",
-    ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,6 +1,5 @@
 import logging
-import os
-from abc import ABC, abstractmethod
+
 
 logger = logging.getLogger(__name__)
 
@@ -28,30 +27,32 @@ def append_doc_link(help_message, path):
     )
 
 
-class CmdBase(ABC):
-    UNINITIALIZED = False
-
+class CmdBase(object):
     def __init__(self, args):
         from dvc.repo import Repo
         from dvc.updater import Updater
 
-        os.chdir(args.cd)
-
-        self.repo = Repo(uninitialized=self.UNINITIALIZED)
+        self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        if self.repo.tmp_dir:
-            hardlink_lock = self.config["core"].get("hardlink_lock", False)
-            updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
-            updater.check()
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
+        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
+        updater.check()
 
-    @abstractmethod
+    @property
+    def default_targets(self):
+        """Default targets for `dvc repro` and `dvc pipeline`."""
+        from dvc.dvcfile import DVC_FILE
+
+        msg = "assuming default target '{}'.".format(DVC_FILE)
+        logger.warning(msg)
+        return [DVC_FILE]
+
+    # Abstract methods that have to be implemented by any inheritance class
     def run(self):
-        pass
+        raise NotImplementedError
 
 
 class CmdBaseNoRepo(CmdBase):
-    def __init__(self, args):  # pylint: disable=super-init-not-called
+    def __init__(self, args):
         self.args = args
-
-        os.chdir(args.cd)

dvc/command/cache.py

@@ -1,31 +1,13 @@
 import argparse
-import logging
 
-from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
-logger = logging.getLogger(__name__)
-
 
 class CmdCacheDir(CmdConfig):
     def run(self):
-        if self.args.value is None and not self.args.unset:
-            if self.args.level:
-                conf = self.config.read(level=self.args.level)
-            else:
-                # Use merged config with default values
-                conf = self.config
-            self._check(conf, False, "cache", "dir")
-            logger.info(conf["cache"]["dir"])
-            return 0
-        with self.config.edit(level=self.args.level) as conf:
-            if self.args.unset:
-                self._check(conf, False, "cache", "dir")
-                del conf["cache"]["dir"]
-            else:
-                self._check(conf, False, "cache")
-                conf["cache"]["dir"] = self.args.value
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
         return 0
 
 
@@ -72,8 +54,6 @@ def add_parser(subparsers, parent_parser):
         "value",
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
-        "config file location. If no path is provided, it returns the "
-        "current cache directory.",
-        nargs="?",
-    ).complete = completion.DIR
+        "config file location.",
+    )
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/check_ignore.py

@@ -1,127 +0,0 @@
-import argparse
-import logging
-
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
-
-logger = logging.getLogger(__name__)
-
-
-class CmdCheckIgnore(CmdBase):
-    def __init__(self, args):
-        super().__init__(args)
-        self.ignore_filter = self.repo.fs.dvcignore
-
-    def _show_results(self, result):
-        if not result.match and not self.args.non_matching:
-            return
-
-        if self.args.details:
-            patterns = result.patterns
-            if not self.args.all:
-                patterns = patterns[-1:]
-
-            for pattern in patterns:
-                logger.info("{}\t{}".format(pattern, result.file))
-        else:
-            logger.info(result.file)
-
-    def _check_one_file(self, target):
-        result = self.ignore_filter.check_ignore(target)
-        self._show_results(result)
-        if result.match:
-            return 0
-        return 1
-
-    def _interactive_mode(self):
-        ret = 1
-        while True:
-            try:
-                target = input()
-            except (KeyboardInterrupt, EOFError):
-                break
-            if target == "":
-                break
-            if not self._check_one_file(target):
-                ret = 0
-        return ret
-
-    def _normal_mode(self):
-        ret = 1
-        for target in self.args.targets:
-            if not self._check_one_file(target):
-                ret = 0
-        return ret
-
-    def _check_args(self):
-        from dvc.exceptions import DvcException
-
-        if not self.args.stdin and not self.args.targets:
-            raise DvcException("`targets` or `--stdin` needed")
-
-        if self.args.stdin and self.args.targets:
-            raise DvcException("cannot have both `targets` and `--stdin`")
-
-        if self.args.non_matching and not self.args.details:
-            raise DvcException(
-                "`--non-matching` is only valid with `--details`"
-            )
-
-        if self.args.all and not self.args.details:
-            raise DvcException("`--all` is only valid with `--details`")
-
-        if self.args.quiet and self.args.details:
-            raise DvcException("cannot use both `--details` and `--quiet`")
-
-    def run(self):
-        self._check_args()
-        if self.args.stdin:
-            return self._interactive_mode()
-        return self._normal_mode()
-
-
-def add_parser(subparsers, parent_parser):
-    ADD_HELP = (
-        "Check whether files or directories are excluded due to `.dvcignore`."
-    )
-
-    parser = subparsers.add_parser(
-        "check-ignore",
-        parents=[parent_parser],
-        description=append_doc_link(ADD_HELP, "check-ignore"),
-        help=ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    parser.add_argument(
-        "-d",
-        "--details",
-        action="store_true",
-        default=False,
-        help="Show the exclude patterns along with each target path.",
-    )
-    parser.add_argument(
-        "-a",
-        "--all",
-        action="store_true",
-        default=False,
-        help="Include the target paths which don’t match any pattern "
-        "in the `--details` list.",
-    )
-    parser.add_argument(
-        "-n",
-        "--non-matching",
-        action="store_true",
-        default=False,
-        help="Include the target paths which don’t match any pattern "
-        "in the `--details` list.",
-    )
-    parser.add_argument(
-        "--stdin",
-        action="store_true",
-        default=False,
-        help="Read paths from standard input instead of providing `targets`.",
-    )
-    parser.add_argument(
-        "targets", nargs="*", help="File or directory paths to check",
-    ).complete = completion.FILE
-    parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -1,16 +1,39 @@
 import argparse
 import logging
-import operator
 
 import colorama
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import CheckoutError
 
 logger = logging.getLogger(__name__)
 
 
+def _human_join(words):
+    words = list(words)
+    if not words:
+        return ""
+
+    return (
+        "{before} and {after}".format(
+            before=", ".join(words[:-1]), after=words[-1],
+        )
+        if len(words) > 1
+        else words[0]
+    )
+
+
+def log_summary(stats):
+    states = ("added", "deleted", "modified")
+    summary = (
+        "{} {}".format(len(stats[state]), state)
+        for state in states
+        if stats.get(state)
+    )
+    logger.info(_human_join(summary) or "No changes.")
+
+
 def log_changes(stats):
     colors = [
         ("modified", colorama.Fore.YELLOW,),
@@ -38,8 +61,6 @@ def log_changes(stats):
 
 class CmdCheckout(CmdBase):
     def run(self):
-        from dvc.utils.humanize import get_summary
-
         stats, exc = None, None
         try:
             stats = self.repo.checkout(
@@ -54,11 +75,7 @@ class CmdCheckout(CmdBase):
             stats = exc.stats
 
         if self.args.summary:
-            default_message = "No changes."
-            msg = get_summary(
-                sorted(stats.items(), key=operator.itemgetter(0))
-            )
-            logger.info(msg or default_message)
+            log_summary(stats)
         else:
             log_changes(stats)
 
@@ -113,9 +130,7 @@ def add_parser(subparsers, parent_parser):
     checkout_parser.add_argument(
         "targets",
         nargs="*",
-        help=(
-            "Limit command scope to these tracked files/directories, "
-            ".dvc files, or stage names."
-        ),
-    ).complete = completion.DVC_FILE
+        help="DVC-files to checkout. Optional. "
+        "(Finds all DVC-files in the workspace by default.)",
+    )
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -1,16 +1,16 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.exceptions import DvcException
+
 
 logger = logging.getLogger(__name__)
 
 
 class CmdCommit(CmdBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         if not self.args.targets:
             self.args.targets = [None]
 
@@ -33,10 +33,7 @@ class CmdCommit(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = (
-        "Record changes to files or directories tracked by DVC"
-        " by storing the current versions in the cache."
-    )
+    COMMIT_HELP = "Save changed data to cache and update DVC-files."
 
     commit_parser = subparsers.add_parser(
         "commit",
@@ -69,7 +66,7 @@ def add_parser(subparsers, parent_parser):
     commit_parser.add_argument(
         "targets",
         nargs="*",
-        help="stages or .dvc files to commit. Optional. "
-        "(Finds all DVC files in the workspace by default.)",
-    ).complete = completion.DVC_FILE
+        help="DVC-files to commit. Optional. "
+        "(Finds all DVC-files in the workspace by default.)",
+    )
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -1,119 +0,0 @@
-import argparse
-import logging
-
-import shtab
-
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
-
-logger = logging.getLogger(__name__)
-FILE = shtab.FILE
-DIR = shtab.DIRECTORY
-
-PREAMBLE = {
-    "bash": """
-# $1=COMP_WORDS[1]
-_dvc_compgen_DVCFiles() {
-  compgen -d -S '/' -- $1  # recurse into subdirs
-  compgen -f -X '!*?.dvc' -- $1
-  compgen -f -X '!*Dvcfile' -- $1
-  compgen -f -X '!*dvc.yaml' -- $1
-}
-
-_dvc_compgen_stages() {
-    local _dvc_stages=($(dvc stage list -q --names-only))
-    compgen -W "${_dvc_stages[*]}" -- $1
-}
-_dvc_compgen_stages_and_files() {
-    _dvc_compgen_DVCFiles $1
-    _dvc_compgen_stages $1
-}
-
-_dvc_compgen_exps() {
-    local _dvc_exps=($(dvc exp list -q --all --names-only))
-    compgen -W "${_dvc_exps[*]}" -- $1
-}
-    """,
-    "zsh": """
-_dvc_compadd_DVCFiles() {
-    _files -g '(*?.dvc|Dvcfile|dvc.yaml)'
-}
-_dvc_compadd_stages() {
-    # this will also show up the description of the stages
-    _describe 'stages' "($(_dvc_stages_output))"
-}
-
-_dvc_stages_output() {
-  dvc stage list -q | awk '{
-    # escape possible `:` on the stage name
-    sub(/:/, "\\\\\\\\:", $1);
-    # read all of the columns except the first
-    # reading `out` from $2, so as not to have a leading whitespace
-    out=$2; for(i=3;i<=NF;i++){out=out" "$i};
-    # print key, ":" and then single-quote the description
-    # colon is a delimiter used by `_describe` to separate field/description
-    print $1":""\\047"out"\\047"
-    # single quote -> \\047
-    }'
-}
-
-_dvc_compadd_stages_and_files() {
-    _dvc_compadd_DVCFiles
-    _dvc_compadd_stages
-}
-
-_dvc_compadd_exps() {
-    _describe 'experiments' "($(dvc exp list -q -a --names-only))"
-}
-    """,
-}
-
-DVC_FILE = {
-    "bash": "_dvc_compgen_DVCFiles",
-    "zsh": "_dvc_compadd_DVCFiles",
-}
-
-STAGE = {
-    "bash": "_dvc_compgen_stages",
-    "zsh": "_dvc_compadd_stages",
-}
-
-DVCFILES_AND_STAGE = {
-    "bash": "_dvc_compgen_stages_and_files",
-    "zsh": "_dvc_compadd_stages_and_files",
-}
-
-EXPERIMENT = {
-    "bash": "_dvc_compgen_exps",
-    "zsh": "_dvc_compadd_exps",
-}
-
-
-class CmdCompletion(CmdBaseNoRepo):
-    def run(self):
-        from dvc.cli import get_main_parser
-
-        parser = get_main_parser()
-        shell = self.args.shell
-        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
-        print(script)
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    COMPLETION_HELP = "Generate shell tab completion."
-    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
-    completion_parser = subparsers.add_parser(
-        "completion",
-        parents=[parent_parser],
-        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
-        help=COMPLETION_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    completion_parser.add_argument(
-        "-s",
-        "--shell",
-        help="Shell syntax for completions.",
-        default="bash",
-        choices=["bash", "zsh"],
-    )
-    completion_parser.set_defaults(func=CmdCompletion)

dvc/command/config.py

@@ -1,115 +1,33 @@
 import argparse
 import logging
-import os
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link, CmdBaseNoRepo
+from dvc.config import Config, ConfigError
 
 logger = logging.getLogger(__name__)
 
-NAME_REGEX = r"^(?P<remote>remote\.)?(?P<section>[^\.]*)\.(?P<option>[^\.]*)$"
-
-
-def _name_type(value):
-    import re
-
-    match = re.match(NAME_REGEX, value)
-    if not match:
-        raise argparse.ArgumentTypeError(
-            "name argument should look like "
-            "remote.name.option or "
-            "section.option"
-        )
-    return (
-        bool(match.group("remote")),
-        match.group("section").lower(),
-        match.group("option").lower(),
-    )
-
 
 class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
-        from dvc.config import Config
-
         super().__init__(args)
 
         self.config = Config(validate=False)
 
     def run(self):
-        if self.args.show_origin:
-            if any((self.args.value, self.args.unset)):
-                logger.error(
-                    "--show-origin can't be used together with any of these "
-                    "options: -u/--unset, value"
-                )
-                return 1
-
-        if self.args.list:
-            return self._list()
-
-        if self.args.name is None:
-            logger.error("name argument is required")
-            return 1
-
-        remote, section, opt = self.args.name
+        section, opt = self.args.name.lower().strip().split(".", 1)
 
         if self.args.value is None and not self.args.unset:
-            return self._get(remote, section, opt)
+            conf = self.config.load_one(self.args.level)
+            self._check(conf, section, opt)
+            logger.info(conf[section][opt])
+            return 0
 
-        return self._set(remote, section, opt)
-
-    def _list(self):
-        if any((self.args.name, self.args.value, self.args.unset)):
-            logger.error(
-                "-l/--list can't be used together with any of these "
-                "options: -u/--unset, name, value"
-            )
-            return 1
-
-        levels = [self.args.level] if self.args.level else self.config.LEVELS
-        for level in levels:
-            conf = self.config.read(level)
-            prefix = self._config_file_prefix(
-                self.args.show_origin, self.config, level
-            )
-            logger.info("\n".join(self._format_config(conf, prefix)))
-
-        return 0
-
-    def _get(self, remote, section, opt):
-        from dvc.config import ConfigError
-
-        levels = (
-            [self.args.level] if self.args.level else self.config.LEVELS[::-1]
-        )
-
-        for level in levels:
-            conf = self.config.read(level)
-            if remote:
-                conf = conf["remote"]
-
-            try:
-                self._check(conf, remote, section, opt)
-            except ConfigError:
-                if self.args.level:
-                    raise
-            else:
-                prefix = self._config_file_prefix(
-                    self.args.show_origin, self.config, level
-                )
-                logger.info("{}{}".format(prefix, conf[section][opt]))
-                break
-
-        return 0
-
-    def _set(self, remote, section, opt):
         with self.config.edit(self.args.level) as conf:
-            if remote:
-                conf = conf["remote"]
             if self.args.unset:
-                self._check(conf, remote, section, opt)
+                self._check(conf, section, opt)
                 del conf[section][opt]
             else:
-                self._check(conf, remote, section)
+                self._check(conf, section)
                 conf[section][opt] = self.args.value
 
         if self.args.name == "cache.type":
@@ -121,72 +39,39 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
-    def _check(self, conf, remote, section, opt=None):
-        from dvc.config import ConfigError
-
-        name = "remote" if remote else "section"
+    def _check(self, conf, section, opt=None):
         if section not in conf:
-            raise ConfigError(f"{name} '{section}' doesn't exist")
+            msg = "section {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
 
         if opt and opt not in conf[section]:
-            raise ConfigError(
-                f"option '{opt}' doesn't exist in {name} '{section}'"
-            )
-
-    @staticmethod
-    def _format_config(config, prefix=""):
-        from dvc.utils.flatten import flatten
-
-        for key, value in flatten(config).items():
-            yield f"{prefix}{key}={value}"
-
-    @staticmethod
-    def _config_file_prefix(show_origin, config, level):
-        from dvc.repo import Repo
-
-        if not show_origin:
-            return ""
-
-        level = level or "repo"
-        fname = config.files[level]
-
-        if level in ["local", "repo"]:
-            fname = os.path.relpath(fname, start=Repo.find_root())
-
-        return fname + "\t"
+            msg = "option {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
 
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
-level_group = parent_config_parser.add_mutually_exclusive_group()
-level_group.add_argument(
+parent_config_parser.add_argument(
     "--global",
     dest="level",
     action="store_const",
     const="global",
     help="Use global config.",
 )
-level_group.add_argument(
+parent_config_parser.add_argument(
     "--system",
     dest="level",
     action="store_const",
     const="system",
     help="Use system config.",
 )
-level_group.add_argument(
-    "--project",
-    dest="level",
-    action="store_const",
-    const="repo",
-    help="Use project config (.dvc/config).",
-)
-level_group.add_argument(
+parent_config_parser.add_argument(
     "--local",
     dest="level",
     action="store_const",
     const="local",
-    help="Use local config (.dvc/config.local).",
+    help="Use local config.",
 )
-parent_config_parser.set_defaults(level=None)
+parent_config_parser.set_defaults(level="repo")
 
 
 def add_parser(subparsers, parent_parser):
@@ -206,24 +91,6 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Unset option.",
     )
-    config_parser.add_argument(
-        "name",
-        nargs="?",
-        type=_name_type,
-        help="Option name (section.option or remote.name.option).",
-    )
+    config_parser.add_argument("name", help="Option name.")
     config_parser.add_argument("value", nargs="?", help="Option value.")
-    config_parser.add_argument(
-        "-l",
-        "--list",
-        default=False,
-        action="store_true",
-        help="List all defined config values.",
-    )
-    config_parser.add_argument(
-        "--show-origin",
-        default=False,
-        action="store_true",
-        help="Show the source file containing each config value.",
-    )
     config_parser.set_defaults(func=CmdConfig)

dvc/command/daemon.py

@@ -1,5 +1,5 @@
-from dvc.command import completion
-from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):
@@ -9,17 +9,15 @@ class CmdDaemonBase(CmdBaseNoRepo):
 class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
-
-        from dvc.config import Config
         from dvc.repo import Repo
+        from dvc.config import Config
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
-        tmp_dir = os.path.join(dvc_dir, "tmp")
         config = Config(dvc_dir, validate=False)
         hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
-        updater = Updater(tmp_dir, hardlink_lock=hardlink_lock)
+        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 
         return 0
@@ -66,7 +64,5 @@ def add_parser(subparsers, parent_parser):
         description=DAEMON_ANALYTICS_HELP,
         help=DAEMON_ANALYTICS_HELP,
     )
-    daemon_analytics_parser.add_argument(
-        "target", help="Analytics file.",
-    ).complete = completion.FILE
+    daemon_analytics_parser.add_argument("target", help="Analytics file.")
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -1,150 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import CmdBase, append_doc_link
-
-logger = logging.getLogger(__name__)
-
-
-def _show_ascii(G):
-    from dvc.dagascii import draw
-    from dvc.repo.graph import get_pipelines
-
-    pipelines = get_pipelines(G)
-
-    ret = []
-    for pipeline in pipelines:
-        ret.append(draw(pipeline.nodes, pipeline.edges))
-
-    return "\n".join(ret)
-
-
-def _show_dot(G):
-    import io
-
-    from networkx.drawing.nx_pydot import write_dot
-
-    dot_file = io.StringIO()
-    write_dot(G, dot_file)
-    return dot_file.getvalue()
-
-
-def _collect_targets(repo, target, outs):
-    if not target:
-        return []
-
-    pairs = repo.stage.collect_granular(target)
-    if not outs:
-        return [stage.addressing for stage, _ in pairs]
-
-    targets = []
-    for stage, info in pairs:
-        if not info:
-            targets.extend([str(out) for out in stage.outs])
-            continue
-
-        for out in repo.outs_trie.itervalues(prefix=info.parts):  # noqa: B301
-            targets.extend(str(out))
-
-    return targets
-
-
-def _transform(repo, outs):
-    import networkx as nx
-
-    from dvc.stage import Stage
-
-    def _relabel(node) -> str:
-        return node.addressing if isinstance(node, Stage) else str(node)
-
-    G = repo.outs_graph if outs else repo.graph
-    return nx.relabel_nodes(G, _relabel, copy=True)
-
-
-def _filter(G, targets, full):
-    import networkx as nx
-
-    if not targets:
-        return G
-
-    H = G.copy()
-    if not full:
-        descendants = set()
-        for target in targets:
-            descendants.update(nx.descendants(G, target))
-            descendants.add(target)
-        H.remove_nodes_from(set(G.nodes()) - descendants)
-
-    undirected = H.to_undirected()
-    connected = set()
-    for target in targets:
-        connected.update(nx.node_connected_component(undirected, target))
-
-    H.remove_nodes_from(set(H.nodes()) - connected)
-
-    return H
-
-
-def _build(repo, target=None, full=False, outs=False):
-    targets = _collect_targets(repo, target, outs)
-    G = _transform(repo, outs)
-    return _filter(G, targets, full)
-
-
-class CmdDAG(CmdBase):
-    def run(self):
-        G = _build(
-            self.repo,
-            target=self.args.target,
-            full=self.args.full,
-            outs=self.args.outs,
-        )
-
-        if self.args.dot:
-            logger.info(_show_dot(G))
-        else:
-            from dvc.utils.pager import pager
-
-            pager(_show_ascii(G))
-
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    DAG_HELP = "Visualize DVC project DAG."
-    dag_parser = subparsers.add_parser(
-        "dag",
-        parents=[parent_parser],
-        description=append_doc_link(DAG_HELP, "dag"),
-        help=DAG_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    dag_parser.add_argument(
-        "--dot",
-        action="store_true",
-        default=False,
-        help="Print DAG with .dot format.",
-    )
-    dag_parser.add_argument(
-        "--full",
-        action="store_true",
-        default=False,
-        help=(
-            "Show full DAG that the target belongs too, instead of "
-            "showing DAG consisting only of ancestors."
-        ),
-    )
-    dag_parser.add_argument(
-        "-o",
-        "--outs",
-        action="store_true",
-        default=False,
-        help="Print output files instead of stages.",
-    )
-    dag_parser.add_argument(
-        "target",
-        nargs="?",
-        help="Stage or output to show pipeline for (optional). "
-        "Finds all stages in the workspace by default.",
-    )
-    dag_parser.set_defaults(func=CmdDAG)

dvc/command/data_sync.py

@@ -1,30 +1,24 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.checkout import log_summary
+from dvc.exceptions import DvcException, CheckoutError
+
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
-    def log_summary(self, stats):
-        from dvc.utils.humanize import get_summary
-
-        default_msg = "Everything is up to date."
-        logger.info(get_summary(stats.items()) or default_msg)
+    @classmethod
+    def check_up_to_date(cls, processed_files_count):
+        if processed_files_count == 0:
+            logger.info("Everything is up to date.")
 
 
 class CmdDataPull(CmdDataBase):
-    def log_summary(self, stats):
-        from dvc.command.checkout import log_changes
-
-        log_changes(stats)
-        super().log_summary(stats)
-
     def run(self):
-        from dvc.exceptions import CheckoutError, DvcException
-
         try:
             stats = self.repo.pull(
                 targets=self.args.targets,
@@ -36,22 +30,19 @@ class CmdDataPull(CmdDataBase):
                 with_deps=self.args.with_deps,
                 force=self.args.force,
                 recursive=self.args.recursive,
-                run_cache=self.args.run_cache,
-                glob=self.args.glob,
             )
-            self.log_summary(stats)
+            log_summary(stats)
         except (CheckoutError, DvcException) as exc:
-            self.log_summary(getattr(exc, "stats", {}))
+            log_summary(getattr(exc, "stats", {}))
             logger.exception("failed to pull data from the cloud")
             return 1
 
+        self.check_up_to_date(stats["downloaded"])
         return 0
 
 
 class CmdDataPush(CmdDataBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             processed_files_count = self.repo.push(
                 targets=self.args.targets,
@@ -62,20 +53,16 @@ class CmdDataPush(CmdDataBase):
                 all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
-                run_cache=self.args.run_cache,
-                glob=self.args.glob,
             )
-            self.log_summary({"pushed": processed_files_count})
         except DvcException:
             logger.exception("failed to push data to the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
 
 
 class CmdDataFetch(CmdDataBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             processed_files_count = self.repo.fetch(
                 targets=self.args.targets,
@@ -86,12 +73,11 @@ class CmdDataFetch(CmdDataBase):
                 all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
-                run_cache=self.args.run_cache,
             )
-            self.log_summary({"fetched": processed_files_count})
         except DvcException:
             logger.exception("failed to fetch data from the cloud")
             return 1
+        self.check_up_to_date(processed_files_count)
         return 0
 
 
@@ -99,30 +85,24 @@ def shared_parent_parser():
     from dvc.cli import get_parent_parser
 
     # Parent parser used in pull/push/status
-    parent_parser = argparse.ArgumentParser(
+    shared_parent_parser = argparse.ArgumentParser(
         add_help=False, parents=[get_parent_parser()]
     )
-    parent_parser.add_argument(
+    shared_parent_parser.add_argument(
         "-j",
         "--jobs",
         type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
+        help="Number of jobs to run simultaneously.",
         metavar="<number>",
     )
-    parent_parser.add_argument(
+    shared_parent_parser.add_argument(
         "targets",
         nargs="*",
-        help=(
-            "Limit command scope to these tracked files/directories, "
-            ".dvc files, or stage names."
-        ),
-    ).complete = completion.DVC_FILE
+        help="Limit command scope to these DVC-files. "
+        "Using -R, directories to search DVC-files in can also be given.",
+    )
 
-    return parent_parser
+    return shared_parent_parser
 
 
 def add_parser(subparsers, _parent_parser):
@@ -182,18 +162,6 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Pull cache for subdirectories of the specified directory.",
     )
-    pull_parser.add_argument(
-        "--run-cache",
-        action="store_true",
-        default=False,
-        help="Fetch run history for all stages.",
-    )
-    pull_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Pull cache for targets matching shell-style wildcards.",
-    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
@@ -243,23 +211,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Push cache for subdirectories of specified directory.",
     )
-    push_parser.add_argument(
-        "--run-cache",
-        action="store_true",
-        default=False,
-        help="Push run history for all stages.",
-    )
-    push_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Allows targets containing shell-style wildcards.",
-    )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
     FETCH_HELP = (
-        "Download files or directories from remote storage to the cache."
+        "Get tracked files or directories from remote storage"
+        " into the cache."
     )
 
     fetch_parser = subparsers.add_parser(
@@ -309,12 +266,6 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for subdirectories of specified directory.",
     )
-    fetch_parser.add_argument(
-        "--run-cache",
-        action="store_true",
-        default=False,
-        help="Fetch run history for all stages.",
-    )
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
@@ -383,18 +334,4 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Show status for all dependencies of the specified target.",
     )
-    status_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Show status of all stages in the specified directory.",
-    )
-    status_parser.add_argument(
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Show status in JSON format.",
-    )
-
     status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -2,15 +2,16 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.exceptions import DvcException
+
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             statement = (
                 "This will destroy all information about your pipelines,"
@@ -26,14 +27,14 @@ class CmdDestroy(CmdBase):
                 )
 
             self.repo.destroy()
-        except DvcException:
+        except Exception:
             logger.exception("failed to destroy DVC")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+    DESTROY_HELP = "Remove DVC-files, local DVC config and data cache."
 
     destroy_parser = subparsers.add_parser(
         "destroy",

dvc/command/diff.py

@@ -5,51 +5,22 @@ import os
 
 import colorama
 
-from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
 
 logger = logging.getLogger(__name__)
 
 
-def _digest(checksum):
-    if isinstance(checksum, str):
-        return checksum[0:8]
-    return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
-
-
-def _show_md(diff, show_hash=False, hide_missing=False):
-    from dvc.utils.diff import table
-
-    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
-    rows = []
-    statuses = ["added", "deleted", "renamed", "modified"]
-    if not hide_missing:
-        statuses.append("not in cache")
-    for status in statuses:
-        entries = diff.get(status, [])
-        if not entries:
-            continue
-        for entry in entries:
-            path = entry["path"]
-            if isinstance(path, dict):
-                path = f"{path['old']} -> {path['new']}"
-            if show_hash:
-                check_sum = _digest(entry.get("hash", ""))
-                rows.append([status, check_sum, path])
-            else:
-                rows.append([status, path])
-
-    return table(header, rows, True)
-
-
 class CmdDiff(CmdBase):
     @staticmethod
-    def _format(diff, hide_missing=False):
+    def _format(diff):
         """
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
 
-        A group's header is colored to enhance readability, for example:
+        A group's header is colored and its entries are sorted to enhance
+        readability, for example:
 
             Added:
                 another_file.txt
@@ -70,21 +41,21 @@ class CmdDiff(CmdBase):
         At the bottom, include a summary with the number of files per state.
         """
 
+        def _digest(checksum):
+            if type(checksum) is str:
+                return checksum[0:8]
+            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
         colors = {
             "added": colorama.Fore.GREEN,
             "modified": colorama.Fore.YELLOW,
             "deleted": colorama.Fore.RED,
-            "renamed": colorama.Fore.GREEN,
-            "not in cache": colorama.Fore.YELLOW,
         }
 
         summary = {}
         groups = []
 
-        states = ["added", "deleted", "renamed", "modified"]
-        if not hide_missing:
-            states.append("not in cache")
-        for state in states:
+        for state in ["added", "deleted", "modified"]:
             summary[state] = 0
             entries = diff[state]
 
@@ -95,8 +66,6 @@ class CmdDiff(CmdBase):
 
             for entry in entries:
                 path = entry["path"]
-                if isinstance(path, dict):
-                    path = f"{path['old']} -> {path['new']}"
                 checksum = entry.get("hash")
                 summary[state] += 1 if not path.endswith(os.sep) else 0
                 content.append(
@@ -104,7 +73,7 @@ class CmdDiff(CmdBase):
                         space="    ",
                         checksum=_digest(checksum) if checksum else "",
                         separator="  " if checksum else "",
-                        path=path,
+                        path=entry["path"],
                     )
                 )
 
@@ -117,51 +86,31 @@ class CmdDiff(CmdBase):
                 )
             )
 
-        if not sum(summary.values()):
-            return None
-
-        fmt = (
+        groups.append(
             "files summary: {added} added, {deleted} deleted,"
-            " {renamed} renamed, {modified} modified"
+            " {modified} modified".format_map(summary)
         )
-        if not hide_missing:
-            fmt += ", {not in cache} not in cache"
-        groups.append(fmt.format_map(summary))
 
         return "\n\n".join(groups)
 
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
-            diff = self.repo.diff(
-                self.args.a_rev, self.args.b_rev, self.args.targets
-            )
-            show_hash = self.args.show_hash
-            hide_missing = self.args.b_rev or self.args.hide_missing
-            if hide_missing:
-                del diff["not in cache"]
+            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
 
-            for key, entries in diff.items():
-                entries = sorted(
-                    entries,
-                    key=lambda entry: entry["path"]["old"]
-                    if isinstance(entry["path"], dict)
-                    else entry["path"],
-                )
-                if not show_hash:
+            if not any(diff.values()):
+                return 0
+
+            if not self.args.show_hash:
+                for _, entries in diff.items():
                     for entry in entries:
                         del entry["hash"]
-                diff[key] = entries
 
             if self.args.show_json:
-                logger.info(json.dumps(diff))
-            elif self.args.show_md:
-                logger.info(_show_md(diff, show_hash, hide_missing))
-            elif diff:
-                output = self._format(diff, hide_missing)
-                if output:
-                    logger.info(output)
+                res = json.dumps(diff)
+            else:
+                res = self._format(diff)
+
+            logger.info(res)
 
         except DvcException:
             logger.exception("failed to get diff")
@@ -181,15 +130,6 @@ def add_parser(subparsers, parent_parser):
         help=DIFF_DESCRIPTION,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help=(
-            "Specific DVC-tracked files to compare. "
-            "Accepts one or more file paths."
-        ),
-        metavar="<paths>",
-    ).complete = completion.FILE
     diff_parser.add_argument(
         "a_rev",
         help="Old Git commit to compare (defaults to HEAD)",
@@ -213,15 +153,4 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
     )
-    diff_parser.add_argument(
-        "--show-md",
-        help="Show tabulated output in the Markdown format (GFM).",
-        action="store_true",
-        default=False,
-    )
-    diff_parser.add_argument(
-        "--hide-missing",
-        help="Hide missing cache file status.",
-        action="store_true",
-    )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/freeze.py

@@ -1,58 +0,0 @@
-import argparse
-import logging
-
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
-
-
-class CmdFreezeBase(CmdBase):
-    def _run(self, func, name):
-        ret = 0
-        for target in self.args.targets:
-            try:
-                func(target)
-            except DvcException:
-                logger.exception(f"failed to {name} '{target}'")
-                ret = 1
-        return ret
-
-
-class CmdFreeze(CmdFreezeBase):
-    def run(self):
-        return self._run(self.repo.freeze, "freeze")
-
-
-class CmdUnfreeze(CmdFreezeBase):
-    def run(self):
-        return self._run(self.repo.unfreeze, "unfreeze")
-
-
-def add_parser(subparsers, parent_parser):
-    FREEZE_HELP = "Freeze stages or .dvc files."
-    freeze_parser = subparsers.add_parser(
-        "freeze",
-        parents=[parent_parser],
-        description=append_doc_link(FREEZE_HELP, "freeze"),
-        help=FREEZE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    freeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to freeze",
-    ).complete = completion.DVC_FILE
-    freeze_parser.set_defaults(func=CmdFreeze)
-
-    UNFREEZE_HELP = "Unfreeze stages or .dvc files."
-    unfreeze_parser = subparsers.add_parser(
-        "unfreeze",
-        parents=[parent_parser],
-        description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
-        help=UNFREEZE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    unfreeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
-    ).complete = completion.DVC_FILE
-    unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -3,7 +3,9 @@ import logging
 import os
 
 import dvc.prompt as prompt
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+
 
 logger = logging.getLogger(__name__)
 
@@ -21,7 +23,7 @@ class CmdGC(CmdBase):
 
         msg = "This will remove all cache except items used in "
 
-        msg += "the workspace"
+        msg += "the working tree"
         if self.args.all_commits:
             msg += " and all git commits"
         elif self.args.all_branches and self.args.all_tags:
@@ -30,8 +32,6 @@ class CmdGC(CmdBase):
             msg += " and all git branches"
         elif self.args.all_tags:
             msg += " and all git tags"
-        elif self.args.all_experiments:
-            msg += " and all experiments"
 
         if self.args.repos:
             msg += " of the current and the following repos:"
@@ -51,7 +51,6 @@ class CmdGC(CmdBase):
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
             all_commits=self.args.all_commits,
-            all_experiments=self.args.all_experiments,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -80,7 +79,7 @@ def add_parser(subparsers, parent_parser):
         "--workspace",
         action="store_true",
         default=False,
-        help="Keep data files used in the current workspace.",
+        help="Keep data files used in the current workspace (working tree).",
     )
     gc_parser.add_argument(
         "-a",
@@ -102,12 +101,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Keep data files for all Git commits.",
     )
-    gc_parser.add_argument(
-        "--all-experiments",
-        action="store_true",
-        default=False,
-        help="Keep data files for all experiments.",
-    )
     gc_parser.add_argument(
         "-c",
         "--cloud",
@@ -132,11 +125,7 @@ def add_parser(subparsers, parent_parser):
         "-j",
         "--jobs",
         type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
+        help="Number of jobs to run simultaneously.",
         metavar="<number>",
     )
     gc_parser.add_argument(

dvc/command/get.py

@@ -1,10 +1,10 @@
 import argparse
 import logging
 
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
-from . import completion
-from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -39,7 +39,6 @@ class CmdGet(CmdBaseNoRepo):
                 path=self.args.path,
                 out=self.args.out,
                 rev=self.args.rev,
-                jobs=self.args.jobs,
             )
             return 0
         except DvcException:
@@ -64,15 +63,15 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
-    ).complete = completion.FILE
+        "path", help="Path to a file or directory within the repository"
+    )
     get_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-    ).complete = completion.DIR
+    )
     get_parser.add_argument(
         "--rev",
         nargs="?",
@@ -85,15 +84,4 @@ def add_parser(subparsers, parent_parser):
         help="Print the storage location (URL) the target data would be "
         "downloaded from, and exit.",
     )
-    get_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
     get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -1,10 +1,10 @@
 import argparse
 import logging
 
+from .base import append_doc_link
+from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
-from . import completion
-from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -14,10 +14,10 @@ class CmdGetUrl(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            Repo.get_url(self.args.url, out=self.args.out, jobs=self.args.jobs)
+            Repo.get_url(self.args.url, out=self.args.out)
             return 0
         except DvcException:
-            logger.exception(f"failed to get '{self.args.url}'")
+            logger.exception("failed to get '{}'".format(self.args.url))
             return 1
 
 
@@ -34,17 +34,6 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to.",
-    ).complete = completion.DIR
-    get_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
+        "out", nargs="?", help="Destination path to put data to."
     )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -19,9 +19,6 @@ class CmdHookBase(CmdBaseNoRepo):
 
         return self._run()
 
-    def _run(self):
-        raise NotImplementedError
-
 
 class CmdPreCommit(CmdHookBase):
     def _run(self):
@@ -31,7 +28,7 @@ class CmdPreCommit(CmdHookBase):
 
 
 class CmdPostCheckout(CmdHookBase):
-    def _run(self):
+    def run(self):
         # when we are running from pre-commit tool, it doesn't provide CLI
         # flags, but instead provides respective env vars that we could use.
         flag = os.environ.get("PRE_COMMIT_CHECKOUT_TYPE")
@@ -55,31 +52,12 @@ class CmdPostCheckout(CmdHookBase):
 
 
 class CmdPrePush(CmdHookBase):
-    def _run(self):
+    def run(self):
         from dvc.main import main
 
         return main(["push"])
 
 
-class CmdMergeDriver(CmdHookBase):
-    def _run(self):
-        from dvc.dvcfile import Dvcfile
-        from dvc.repo import Repo
-
-        dvc = Repo()
-
-        try:
-            ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
-            our = Dvcfile(dvc, self.args.our, verify=False)
-            their = Dvcfile(dvc, self.args.their, verify=False)
-
-            our.merge(ancestor, their)
-
-            return 0
-        finally:
-            dvc.close()
-
-
 def add_parser(subparsers, parent_parser):
     GIT_HOOK_HELP = "Run GIT hook."
 
@@ -132,27 +110,3 @@ def add_parser(subparsers, parent_parser):
         "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
-
-    MERGE_DRIVER_HELP = "Run GIT merge driver."
-    merge_driver_parser = git_hook_subparsers.add_parser(
-        "merge-driver",
-        parents=[parent_parser],
-        description=MERGE_DRIVER_HELP,
-        help=MERGE_DRIVER_HELP,
-    )
-    merge_driver_parser.add_argument(
-        "--ancestor",
-        required=True,
-        help="Ancestor's version of the conflicting file.",
-    )
-    merge_driver_parser.add_argument(
-        "--our",
-        required=True,
-        help="Current version of the conflicting file.",
-    )
-    merge_driver_parser.add_argument(
-        "--their",
-        required=True,
-        help="Other branch's version of the conflicting file.",
-    )
-    merge_driver_parser.set_defaults(func=CmdMergeDriver)

dvc/command/imp.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -15,11 +16,7 @@ class CmdImport(CmdBase):
                 self.args.url,
                 self.args.path,
                 out=self.args.out,
-                fname=self.args.file,
                 rev=self.args.rev,
-                no_exec=self.args.no_exec,
-                desc=self.args.desc,
-                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception(
@@ -48,50 +45,19 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
-    ).complete = completion.FILE
+        "path", help="Path to a file or directory within the repository"
+    )
     import_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
-    ).complete = completion.DIR
+    )
     import_parser.add_argument(
         "--rev",
         nargs="?",
         help="Git revision (e.g. SHA, branch, tag)",
         metavar="<commit>",
     )
-    import_parser.add_argument(
-        "--file",
-        help="Specify name of the .dvc file this command will generate.",
-        metavar="<filename>",
-    )
-    import_parser.add_argument(
-        "--no-exec",
-        action="store_true",
-        default=False,
-        help="Only create .dvc file without actually downloading it.",
-    )
-    import_parser.add_argument(
-        "--desc",
-        type=str,
-        metavar="<text>",
-        help=(
-            "User description of the data (optional). "
-            "This doesn't affect any DVC operations."
-        ),
-    )
-    import_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -12,14 +13,7 @@ class CmdImportUrl(CmdBase):
     def run(self):
         try:
             self.repo.imp_url(
-                self.args.url,
-                out=self.args.out,
-                fname=self.args.file,
-                no_exec=self.args.no_exec,
-                remote=self.args.remote,
-                to_remote=self.args.to_remote,
-                desc=self.args.desc,
-                jobs=self.args.jobs,
+                self.args.url, out=self.args.out, fname=self.args.file
             )
         except DvcException:
             logger.exception(
@@ -46,61 +40,28 @@ def add_parser(subparsers, parent_parser):
     )
     import_parser.add_argument(
         "url",
-        help="Location of the data to download. Supported URLs:\n"
-        "/absolute/path/to/file/or/dir\n"
-        "relative/path/to/file/or/dir\n"
-        "C:\\\\path\\to\\file\\or\\dir\n"
+        help="Supported urls:\n"
+        "/path/to/file\n"
+        "/path/to/directory\n"
+        "C:\\\\path\\to\\file\n"
+        "C:\\\\path\\to\\directory\n"
         "https://example.com/path/to/file\n"
-        "s3://bucket/key/path\n"
-        "gs://bucket/path/to/file/or/dir\n"
+        "s3://bucket/path/to/file\n"
+        "s3://bucket/path/to/directory\n"
+        "gs://bucket/path/to/file\n"
+        "gs://bucket/path/to/directory\n"
         "hdfs://example.com/path/to/file\n"
-        "ssh://example.com/absolute/path/to/file/or/dir\n"
-        "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
+        "ssh://example.com:/path/to/file\n"
+        "ssh://example.com:/path/to/directory\n"
+        "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to.",
-    ).complete = completion.DIR
+        "out", nargs="?", help="Destination path to put files to."
+    )
     import_parser.add_argument(
+        "-f",
         "--file",
-        help="Specify name of the .dvc file this command will generate.",
+        help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
-    ).complete = completion.DIR
-    import_parser.add_argument(
-        "--no-exec",
-        action="store_true",
-        default=False,
-        help="Only create .dvc file without actually downloading it.",
-    )
-    import_parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Download it directly to the remote",
-    )
-    import_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to download to",
-        metavar="<name>",
-    )
-    import_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
-    import_parser.add_argument(
-        "--desc",
-        type=str,
-        metavar="<text>",
-        help=(
-            "User description of the data (optional). "
-            "This doesn't affect any DVC operations."
-        ),
     )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/init.py

@@ -1,42 +1,17 @@
 import argparse
 import logging
 
-import colorama
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 
-from dvc import analytics
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.utils import boxify
-from dvc.utils import format_link as fmt_link
 
 logger = logging.getLogger(__name__)
 
 
-def _welcome_message():
-    if analytics.is_enabled():
-        logger.info(
-            boxify(
-                "DVC has enabled anonymous aggregate usage analytics.\n"
-                "Read the analytics documentation (and how to opt-out) here:\n"
-                + fmt_link("https://dvc.org/doc/user-guide/analytics"),
-                border_color="red",
-            )
-        )
-
-    msg = (
-        "{yellow}What's next?{nc}\n"
-        "{yellow}------------{nc}\n"
-        f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
-        f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
-        f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
-    ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
-
-    logger.info(msg)
-
-
 class CmdInit(CmdBaseNoRepo):
     def run(self):
-        from dvc.exceptions import InitError
         from dvc.repo import Repo
+        from dvc.exceptions import InitError
 
         try:
             self.repo = Repo.init(
@@ -46,7 +21,6 @@ class CmdInit(CmdBaseNoRepo):
                 subdir=self.args.subdir,
             )
             self.config = self.repo.config
-            _welcome_message()
         except InitError:
             logger.exception("failed to initiate DVC")
             return 1

dvc/command/install.py

@@ -1,8 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+
 
 logger = logging.getLogger(__name__)
 
@@ -11,7 +12,7 @@ class CmdInstall(CmdBase):
     def run(self):
         try:
             self.repo.install(self.args.use_pre_commit_tool)
-        except DvcException:
+        except Exception:
             logger.exception("failed to install DVC Git hooks")
             return 1
         return 0

dvc/command/live.py

@@ -1,94 +0,0 @@
-import argparse
-import logging
-import os
-
-from dvc.command import completion
-from dvc.command.base import CmdBase, fix_subparsers
-from dvc.utils.html import write
-
-logger = logging.getLogger(__name__)
-
-
-class CmdLive(CmdBase):
-    UNINITIALIZED = True
-
-    def _run(self, target, revs=None):
-        metrics, plots = self.repo.live.show(target=target, revs=revs)
-
-        html_path = self.args.target + ".html"
-        write(html_path, plots, metrics)
-
-        logger.info(f"\nfile://{os.path.abspath(html_path)}")
-
-        return 0
-
-
-class CmdLiveShow(CmdLive):
-    def run(self):
-        return self._run(self.args.target)
-
-
-class CmdLiveDiff(CmdLive):
-    def run(self):
-        return self._run(self.args.target, self.args.revs)
-
-
-def shared_parent_parser():
-    parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument(
-        "target", help="Logs dir to produce summary from",
-    ).complete = completion.DIR
-    parent_parser.add_argument(
-        "-o",
-        "--out",
-        default=None,
-        help="Destination path to save plots to",
-        metavar="<path>",
-    ).complete = completion.DIR
-    return parent_parser
-
-
-def add_parser(subparsers, parent_parser):
-    LIVE_DESCRIPTION = (
-        "Commands to visualize and compare dvclive-produced logs."
-    )
-    live_parser = subparsers.add_parser(
-        "live",
-        parents=[parent_parser],
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        description=LIVE_DESCRIPTION,
-    )
-    live_subparsers = live_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc live CMD --help` to display command-specific help.",
-    )
-
-    fix_subparsers(live_subparsers)
-
-    SHOW_HELP = "Visualize dvclive directory content."
-    live_show_parser = live_subparsers.add_parser(
-        "show",
-        parents=[parent_parser, shared_parent_parser()],
-        help=SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    live_show_parser.set_defaults(func=CmdLiveShow)
-
-    DIFF_HELP = (
-        "Show multiple versions of dvclive data, "
-        "by plotting it in single view."
-    )
-    live_diff_parser = live_subparsers.add_parser(
-        "diff",
-        parents=[parent_parser, shared_parent_parser()],
-        help=DIFF_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    live_diff_parser.add_argument(
-        "--revs",
-        nargs="*",
-        default=None,
-        help="Git revision (e.g. SHA, branch, tag)",
-        metavar="<commit>",
-    )
-    live_diff_parser.set_defaults(func=CmdLiveDiff)

dvc/command/lock.py

@@ -0,0 +1,61 @@
+import argparse
+import logging
+
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.exceptions import DvcException
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdLockBase(CmdBase):
+    def _run(self, unlock):
+        for target in self.args.targets:
+            try:
+                self.repo.lock_stage(target, unlock=unlock)
+            except DvcException:
+                logger.exception(
+                    "failed to {}lock '{}'".format(
+                        "un" if unlock else "", target
+                    )
+                )
+
+                return 1
+        return 0
+
+
+class CmdLock(CmdLockBase):
+    def run(self):
+        return self._run(False)
+
+
+class CmdUnlock(CmdLockBase):
+    def run(self):
+        return self._run(True)
+
+
+def add_parser(subparsers, parent_parser):
+    LOCK_HELP = "Lock DVC-files."
+    lock_parser = subparsers.add_parser(
+        "lock",
+        parents=[parent_parser],
+        description=append_doc_link(LOCK_HELP, "lock"),
+        help=LOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    lock_parser.add_argument("targets", nargs="+", help="DVC-files to lock.")
+    lock_parser.set_defaults(func=CmdLock)
+
+    UNLOCK_HELP = "Unlock DVC-files."
+    unlock_parser = subparsers.add_parser(
+        "unlock",
+        parents=[parent_parser],
+        description=append_doc_link(UNLOCK_HELP, "unlock"),
+        help=UNLOCK_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    unlock_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to unlock."
+    )
+    unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/ls/__init__.py

@@ -2,11 +2,12 @@ import argparse
 import logging
 import sys
 
-from dvc.command import completion
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -32,14 +33,14 @@ class CmdList(CmdBaseNoRepo):
                 self.args.path,
                 rev=self.args.rev,
                 recursive=self.args.recursive,
-                dvc_only=self.args.dvc_only,
+                outs_only=self.args.outs_only,
             )
             if entries:
                 entries = _prettify(entries, sys.stdout.isatty())
                 logger.info("\n".join(entries))
             return 0
         except DvcException:
-            logger.exception(f"failed to list '{self.args.url}'")
+            logger.exception("failed to list '{}'".format(self.args.url))
             return 1
 
 
@@ -63,7 +64,7 @@ def add_parser(subparsers, parent_parser):
         help="Recursively list files.",
     )
     list_parser.add_argument(
-        "--dvc-only", action="store_true", help="Show only DVC outputs."
+        "--outs-only", action="store_true", help="Show only DVC outputs."
     )
     list_parser.add_argument(
         "--rev",
@@ -75,5 +76,5 @@ def add_parser(subparsers, parent_parser):
         "path",
         nargs="?",
         help="Path to directory within the repository to list outputs for",
-    ).complete = completion.DIR
+    )
     list_parser.set_defaults(func=CmdList)

dvc/command/ls/ls_colors.py

@@ -1,7 +1,7 @@
 import os
 
 
-class LsColors:
+class LsColors(object):
     default = "rs=0:di=01;34:ex=01;32"
 
     def __init__(self, lscolors=None):
@@ -45,4 +45,4 @@ class LsColors:
         if not val:
             return text
         rs = self._codes.get("rs", 0)
-        return f"\033[{val}m{text}\033[{rs}m"
+        return "\033[{}m{}\033[{}m".format(val, text, rs)

dvc/command/metrics.py

@@ -1,79 +1,44 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.exceptions import BadMetricError, DvcException
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
+from dvc.exceptions import BadMetricError
+from dvc.exceptions import DvcException
+
 
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_PRECISION = 5
-
-
-def _show_metrics(
-    metrics,
-    markdown=False,
-    all_branches=False,
-    all_tags=False,
-    all_commits=False,
-    precision=None,
+def show_metrics(
+    metrics, all_branches=False, all_tags=False, all_commits=False
 ):
-    from dvc.utils.diff import format_dict, table
-    from dvc.utils.flatten import flatten
+    from flatten_json import flatten
+    from dvc.utils.diff import format_dict
 
     # When `metrics` contains a `None` key, it means that some files
     # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
     missing = metrics.pop(None, None)
-    with_rev = any([all_branches, all_tags, all_commits])
-    header_set = set()
-    rows = []
 
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-        return val
-
-    for _branch, val in metrics.items():
-        for _fname, metric in val.items():
-            if not isinstance(metric, dict):
-                header_set.add("")
-                continue
-            for key, _val in flatten(format_dict(metric)).items():
-                header_set.add(key)
-    header = sorted(header_set)
     for branch, val in metrics.items():
-        for fname, metric in val.items():
-            row = []
-            if with_rev:
-                row.append(branch)
-            row.append(fname)
-            if not isinstance(metric, dict):
-                row.append(str(metric))
-                rows.append(row)
-                continue
-            flattened_val = flatten(format_dict(metric))
+        if all_branches or all_tags or all_commits:
+            logger.info("{branch}:".format(branch=branch))
 
-            for i in header:
-                row.append(_round(flattened_val.get(i)))
-            rows.append(row)
-    header.insert(0, "Path")
-    if with_rev:
-        header.insert(0, "Revision")
+        for fname, metric in val.items():
+            if not isinstance(metric, dict):
+                logger.info("\t{}: {}".format(fname, str(metric)))
+                continue
+
+            logger.info("\t{}:".format(fname))
+            for key, value in flatten(format_dict(metric), ".").items():
+                logger.info("\t\t{}: {}".format(key, value))
 
     if missing:
         raise BadMetricError(missing)
-    return table(header, rows, markdown)
 
 
-class CmdMetricsBase(CmdBase):
-    UNINITIALIZED = True
-
-
-class CmdMetricsShow(CmdMetricsBase):
+class CmdMetricsShow(CmdBase):
     def run(self):
         try:
             metrics = self.repo.metrics.show(
@@ -89,56 +54,65 @@ class CmdMetricsShow(CmdMetricsBase):
 
                 logger.info(json.dumps(metrics))
             else:
-                table = _show_metrics(
+                show_metrics(
                     metrics,
-                    self.args.show_md,
                     self.args.all_branches,
                     self.args.all_tags,
                     self.args.all_commits,
                 )
-                if table:
-                    logger.info(table)
         except DvcException:
-            logger.exception("")
+            logger.exception("failed to show metrics")
             return 1
 
         return 0
 
 
-def _show_diff(diff, markdown=False, no_path=False, precision=None):
+class CmdMetricsAdd(CmdBase):
+    def run(self):
+        try:
+            self.repo.metrics.add(self.args.path)
+        except DvcException:
+            msg = "failed to add metric file '{}'".format(self.args.path)
+            logger.exception(msg)
+            return 1
+
+        return 0
+
+
+class CmdMetricsRemove(CmdBase):
+    def run(self):
+        try:
+            self.repo.metrics.remove(self.args.path)
+        except DvcException:
+            msg = "failed to remove metric file '{}'".format(self.args.path)
+            logger.exception(msg)
+            return 1
+
+        return 0
+
+
+def _show_diff(diff):
     from collections import OrderedDict
 
     from dvc.utils.diff import table
 
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-
-        return val
-
     rows = []
     for fname, mdiff in diff.items():
         sorted_mdiff = OrderedDict(sorted(mdiff.items()))
         for metric, change in sorted_mdiff.items():
-            row = [] if no_path else [fname]
-            row.append(metric)
-            row.append(_round(change.get("old")))
-            row.append(_round(change["new"]))
-            row.append(_round(change.get("diff")))
-            rows.append(row)
+            rows.append(
+                [
+                    fname,
+                    metric,
+                    change["new"],
+                    change.get("diff", "diff not supported"),
+                ]
+            )
 
-    header = [] if no_path else ["Path"]
-    header.append("Metric")
-    header.extend(["Old", "New"])
-    header.append("Change")
-
-    return table(header, rows, markdown)
+    return table(["Path", "Metric", "Value", "Change"], rows)
 
 
-class CmdMetricsDiff(CmdMetricsBase):
+class CmdMetricsDiff(CmdBase):
     def run(self):
         try:
             diff = self.repo.metrics.diff(
@@ -154,12 +128,7 @@ class CmdMetricsDiff(CmdMetricsBase):
 
                 logger.info(json.dumps(diff))
             else:
-                table = _show_diff(
-                    diff,
-                    self.args.show_md,
-                    self.args.no_path,
-                    precision=self.args.precision,
-                )
+                table = _show_diff(diff)
                 if table:
                     logger.info(table)
 
@@ -171,7 +140,7 @@ class CmdMetricsDiff(CmdMetricsBase):
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Commands to display and compare metrics."
+    METRICS_HELP = "Commands to add, manage, collect, and display metrics."
 
     metrics_parser = subparsers.add_parser(
         "metrics",
@@ -188,6 +157,17 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(metrics_subparsers)
 
+    METRICS_ADD_HELP = "Mark a DVC-tracked file as a metric."
+    metrics_add_parser = metrics_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
+        help=METRICS_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_add_parser.add_argument("path", help="Path to a metric file.")
+    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
+
     METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
@@ -199,11 +179,8 @@ def add_parser(subparsers, parent_parser):
     metrics_show_parser.add_argument(
         "targets",
         nargs="*",
-        help=(
-            "Limit command scope to these metrics files. Using -R, "
-            "directories to search metrics files in can also be given."
-        ),
-    ).complete = completion.FILE
+        help="Metric files or directories (see -R) to show",
+    )
     metrics_show_parser.add_argument(
         "-a",
         "--all-branches",
@@ -224,18 +201,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show metrics for all commits.",
     )
-    metrics_show_parser.add_argument(
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Show output in JSON format.",
-    )
-    metrics_show_parser.add_argument(
-        "--show-md",
-        action="store_true",
-        default=False,
-        help="Show tabulated output in the Markdown format (GFM).",
-    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -243,24 +208,19 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=(
             "If any target is a directory, recursively search and process "
-            "metrics files."
+            "metric files."
         ),
     )
     metrics_show_parser.add_argument(
-        "--precision",
-        type=int,
-        help=(
-            "Round metrics to `n` digits precision after the decimal point. "
-            f"Rounds to {DEFAULT_PRECISION} digits by default."
-        ),
-        metavar="<n>",
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
     )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
-    METRICS_DIFF_HELP = (
-        "Show changes in metrics between commits in the DVC repository, or "
-        "between a commit and the workspace."
-    )
+    METRICS_DIFF_HELP = "Show changes in metrics between commits"
+    " in the DVC repository, or between a commit and the workspace."
     metrics_diff_parser = metrics_subparsers.add_parser(
         "diff",
         parents=[parent_parser],
@@ -274,20 +234,17 @@ def add_parser(subparsers, parent_parser):
     metrics_diff_parser.add_argument(
         "b_rev",
         nargs="?",
-        help="New Git commit to compare (defaults to the current workspace)",
+        help=("New Git commit to compare (defaults to the current workspace)"),
     )
     metrics_diff_parser.add_argument(
         "--targets",
         nargs="*",
         help=(
-            "Specific metrics file(s) to compare "
-            "(even if not found as `metrics` in `dvc.yaml`). "
-            "Using -R, directories to search metrics files in "
-            "can also be given."
-            "Shows all tracked metrics by default."
+            "Metric files or directories (see -R) to show diff for. "
+            "Shows diff for all metric files by default."
         ),
         metavar="<paths>",
-    ).complete = completion.FILE
+    )
     metrics_diff_parser.add_argument(
         "-R",
         "--recursive",
@@ -295,7 +252,7 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=(
             "If any target is a directory, recursively search and process "
-            "metrics files."
+            "metric files."
         ),
     )
     metrics_diff_parser.add_argument(
@@ -310,25 +267,15 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show output in JSON format.",
     )
-    metrics_diff_parser.add_argument(
-        "--show-md",
-        action="store_true",
-        default=False,
-        help="Show tabulated output in the Markdown format (GFM).",
-    )
-    metrics_diff_parser.add_argument(
-        "--no-path",
-        action="store_true",
-        default=False,
-        help="Don't show metric path.",
-    )
-    metrics_diff_parser.add_argument(
-        "--precision",
-        type=int,
-        help=(
-            "Round metrics to `n` digits precision after the decimal point. "
-            f"Rounds to {DEFAULT_PRECISION} digits by default."
-        ),
-        metavar="<n>",
-    )
     metrics_diff_parser.set_defaults(func=CmdMetricsDiff)
+
+    METRICS_REMOVE_HELP = "Remove metric mark on a DVC-tracked file."
+    metrics_remove_parser = metrics_subparsers.add_parser(
+        "remove",
+        parents=[parent_parser],
+        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
+        help=METRICS_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
+    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/move.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -25,7 +26,7 @@ def add_parser(subparsers, parent_parser):
     MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
     MOVE_DESCRIPTION = (
         "Rename or move a DVC controlled data file or a directory.\n"
-        "It renames and modifies the corresponding .dvc file to reflect the"
+        "It renames and modifies the corresponding DVC-file to reflect the"
         " changes."
     )
 
@@ -37,9 +38,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory.",
-    ).complete = completion.FILE
-    move_parser.add_argument(
-        "dst", help="Destination path.",
-    ).complete = completion.FILE
+        "src", help="Source path to a data file or directory."
+    )
+    move_parser.add_argument("dst", help="Destination path.")
     move_parser.set_defaults(func=CmdMove)

dvc/command/params.py

@@ -1,44 +1,35 @@
 import argparse
 import logging
+
 from collections import OrderedDict
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.base import fix_subparsers
 from dvc.exceptions import DvcException
 
+
 logger = logging.getLogger(__name__)
 
 
-def _show_diff(diff, markdown=False, no_path=False):
+def _show_diff(diff):
     from dvc.utils.diff import table
 
     rows = []
     for fname, pdiff in diff.items():
         sorted_pdiff = OrderedDict(sorted(pdiff.items()))
         for param, change in sorted_pdiff.items():
-            row = [] if no_path else [fname]
-            row.append(param)
-            row.append(change["old"])
-            row.append(change["new"])
-            rows.append(row)
+            rows.append([fname, param, change["old"], change["new"]])
 
-    header = [] if no_path else ["Path"]
-    header.append("Param")
-    header.append("Old")
-    header.append("New")
-
-    return table(header, rows, markdown)
+    return table(["Path", "Param", "Old", "New"], rows)
 
 
 class CmdParamsDiff(CmdBase):
-    UNINITIALIZED = True
-
     def run(self):
         try:
             diff = self.repo.params.diff(
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
-                targets=self.args.targets,
                 all=self.args.all,
             )
 
@@ -47,7 +38,7 @@ class CmdParamsDiff(CmdBase):
 
                 logger.info(json.dumps(diff))
             else:
-                table = _show_diff(diff, self.args.show_md, self.args.no_path)
+                table = _show_diff(diff)
                 if table:
                     logger.info(table)
 
@@ -95,16 +86,6 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help=("New Git commit to compare (defaults to the current workspace)"),
     )
-    params_diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help=(
-            "Specific params file(s) to compare "
-            "(even if not found as `params` in `dvc.yaml`). "
-            "Shows all tracked params by default."
-        ),
-        metavar="<paths>",
-    ).complete = completion.FILE
     params_diff_parser.add_argument(
         "--all",
         action="store_true",
@@ -117,16 +98,4 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show output in JSON format.",
     )
-    params_diff_parser.add_argument(
-        "--show-md",
-        action="store_true",
-        default=False,
-        help="Show tabulated output in the Markdown format (GFM).",
-    )
-    params_diff_parser.add_argument(
-        "--no-path",
-        action="store_true",
-        default=False,
-        help="Don't show params path.",
-    )
     params_diff_parser.set_defaults(func=CmdParamsDiff)

dvc/command/pipeline.py

@@ -0,0 +1,245 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+
+
+logger = logging.getLogger(__name__)
+
+
+class CmdPipelineShow(CmdBase):
+    def _show(self, target, commands, outs, locked):
+        import networkx
+        from dvc import dvcfile
+        from dvc.utils import parse_target
+
+        path, name = parse_target(target)
+        stage = dvcfile.Dvcfile(self.repo, path).stages[name]
+        G = self.repo.graph
+        stages = networkx.dfs_postorder_nodes(G, stage)
+        if locked:
+            stages = [s for s in stages if s.locked]
+
+        for stage in stages:
+            if commands:
+                if stage.cmd is None:
+                    continue
+                logger.info(stage.cmd)
+            elif outs:
+                for out in stage.outs:
+                    logger.info(str(out))
+            else:
+                logger.info(stage.addressing)
+
+    def _build_graph(self, target, commands=False, outs=False):
+        import networkx
+        from dvc import dvcfile
+        from dvc.repo.graph import get_pipeline
+        from dvc.utils import parse_target
+
+        path, name = parse_target(target)
+        target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
+        G = get_pipeline(self.repo.pipelines, target_stage)
+
+        nodes = set()
+        for stage in networkx.dfs_preorder_nodes(G, target_stage):
+            if commands:
+                if stage.cmd is None:
+                    continue
+                nodes.add(stage.cmd)
+            elif outs:
+                for out in stage.outs:
+                    nodes.add(str(out))
+            else:
+                nodes.add(stage.addressing)
+
+        edges = []
+        for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
+            if commands:
+                if to_stage.cmd is None:
+                    continue
+                edges.append((from_stage.cmd, to_stage.cmd))
+            elif outs:
+                for from_out in from_stage.outs:
+                    for to_out in to_stage.outs:
+                        edges.append((str(from_out), str(to_out)))
+            else:
+                edges.append((from_stage.addressing, to_stage.addressing))
+
+        return list(nodes), edges, networkx.is_tree(G)
+
+    def _show_ascii(self, target, commands, outs):
+        from dvc.dagascii import draw
+
+        nodes, edges, _ = self._build_graph(target, commands, outs)
+
+        if not nodes:
+            return
+
+        draw(nodes, edges)
+
+    def _show_dependencies_tree(self, target, commands, outs):
+        from treelib import Tree
+
+        nodes, edges, is_tree = self._build_graph(target, commands, outs)
+        if not nodes:
+            return
+        if not is_tree:
+            raise DvcException(
+                "DAG is not a tree, can not print it in tree-structure way, "
+                "please use --ascii instead"
+            )
+
+        tree = Tree()
+        tree.create_node(target, target)  # Root node
+        observe_list = [target]
+        while len(observe_list) > 0:
+            current_root = observe_list[0]
+            for edge in edges:
+                if edge[0] == current_root:
+                    tree.create_node(edge[1], edge[1], parent=current_root)
+                    observe_list.append(edge[1])
+            observe_list.pop(0)
+        tree.show()
+
+    def _write_dot(self, target, commands, outs):
+        import io
+        import networkx
+        from networkx.drawing.nx_pydot import write_dot
+
+        _, edges, _ = self._build_graph(target, commands, outs)
+        edges = [edge[::-1] for edge in edges]
+
+        simple_g = networkx.DiGraph()
+        simple_g.add_edges_from(edges)
+
+        dot_file = io.StringIO()
+        write_dot(simple_g, dot_file)
+        logger.info(dot_file.getvalue())
+
+    def run(self):
+        if not self.args.targets:
+            self.args.targets = self.default_targets
+
+        for target in self.args.targets:
+            try:
+                if self.args.ascii:
+                    self._show_ascii(
+                        target, self.args.commands, self.args.outs
+                    )
+                elif self.args.dot:
+                    self._write_dot(target, self.args.commands, self.args.outs)
+                elif self.args.tree:
+                    self._show_dependencies_tree(
+                        target, self.args.commands, self.args.outs
+                    )
+                else:
+                    self._show(
+                        target,
+                        self.args.commands,
+                        self.args.outs,
+                        self.args.locked,
+                    )
+            except DvcException:
+                msg = "failed to show pipeline for '{}'".format(target)
+                logger.exception(msg)
+                return 1
+        return 0
+
+
+class CmdPipelineList(CmdBase):
+    def run(self):
+        pipelines = self.repo.pipelines
+        for pipeline in pipelines:
+            for stage in pipeline:
+                logger.info(stage.addressing)
+            if len(pipeline) != 0:
+                logger.info("=" * 80)
+        logger.info("{} pipelines total".format(len(pipelines)))
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PIPELINE_HELP = "Manage pipelines."
+    pipeline_parser = subparsers.add_parser(
+        "pipeline",
+        parents=[parent_parser],
+        description=append_doc_link(PIPELINE_HELP, "pipeline"),
+        help=PIPELINE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    pipeline_subparsers = pipeline_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc pipeline CMD --help` for command-specific help.",
+    )
+
+    fix_subparsers(pipeline_subparsers)
+
+    PIPELINE_LIST_HELP = "List connected groups of stages (pipelines)."
+    pipeline_list_parser = pipeline_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
+        help=PIPELINE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pipeline_list_parser.set_defaults(func=CmdPipelineList)
+
+    PIPELINE_SHOW_HELP = "Show stages in a pipeline."
+    pipeline_show_parser = pipeline_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
+        help=PIPELINE_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
+    pipeline_show_group.add_argument(
+        "-c",
+        "--commands",
+        action="store_true",
+        default=False,
+        help="Print commands instead of paths to DVC-files.",
+    )
+    pipeline_show_group.add_argument(
+        "-o",
+        "--outs",
+        action="store_true",
+        default=False,
+        help="Print output files instead of paths to DVC-files.",
+    )
+    pipeline_show_parser.add_argument(
+        "-l",
+        "--locked",
+        action="store_true",
+        default=False,
+        help="Print locked DVC stages",
+    )
+    pipeline_show_parser.add_argument(
+        "--ascii",
+        action="store_true",
+        default=False,
+        help="Output DAG as ASCII.",
+    )
+    pipeline_show_parser.add_argument(
+        "--dot",
+        action="store_true",
+        default=False,
+        help="Print DAG with .dot format.",
+    )
+    pipeline_show_parser.add_argument(
+        "--tree",
+        action="store_true",
+        default=False,
+        help="Output DAG as Dependencies Tree.",
+    )
+    pipeline_show_parser.add_argument(
+        "targets",
+        nargs="*",
+        help="DVC-files to show pipeline for. Optional. "
+        "(Finds all DVC-files in the workspace by default.)",
+    )
+    pipeline_show_parser.set_defaults(func=CmdPipelineShow)

dvc/command/plots.py

@@ -1,231 +0,0 @@
-import argparse
-import logging
-import os
-
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.exceptions import DvcException
-from dvc.utils import format_link
-from dvc.utils.html import write
-
-logger = logging.getLogger(__name__)
-
-
-class CmdPlots(CmdBase):
-    def _func(self, *args, **kwargs):
-        raise NotImplementedError
-
-    def _props(self):
-        from dvc.schema import PLOT_PROPS
-
-        # Pass only props specified by user, to not shadow ones from plot def
-        props = {p: getattr(self.args, p) for p in PLOT_PROPS}
-        return {k: v for k, v in props.items() if v is not None}
-
-    def run(self):
-        if self.args.show_vega:
-            if not self.args.targets:
-                logger.error("please specify a target for `--show-vega`")
-                return 1
-            if len(self.args.targets) > 1:
-                logger.error(
-                    "you can only specify one target for `--show-vega`"
-                )
-                return 1
-
-        try:
-            plots = self._func(targets=self.args.targets, props=self._props())
-
-            if self.args.show_vega:
-                target = self.args.targets[0]
-                logger.info(plots[target])
-                return 0
-
-            path = self.args.out or "plots.html"
-            path = os.path.join(os.getcwd(), path)
-
-            write(path, plots)
-
-            logger.info(f"file://{path}")
-
-        except DvcException:
-            logger.exception("")
-            return 1
-
-        return 0
-
-
-class CmdPlotsShow(CmdPlots):
-    UNINITIALIZED = True
-
-    def _func(self, *args, **kwargs):
-        return self.repo.plots.show(*args, **kwargs)
-
-
-class CmdPlotsDiff(CmdPlots):
-    UNINITIALIZED = True
-
-    def _func(self, *args, **kwargs):
-        return self.repo.plots.diff(
-            *args,
-            revs=self.args.revisions,
-            experiment=self.args.experiment,
-            **kwargs,
-        )
-
-
-class CmdPlotsModify(CmdPlots):
-    def run(self):
-        self.repo.plots.modify(
-            self.args.target, props=self._props(), unset=self.args.unset,
-        )
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    PLOTS_HELP = (
-        "Commands to visualize and compare plot metrics in structured files "
-        "(JSON, YAML, CSV, TSV)"
-    )
-
-    plots_parser = subparsers.add_parser(
-        "plots",
-        parents=[parent_parser],
-        description=append_doc_link(PLOTS_HELP, "plots"),
-        help=PLOTS_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    plots_subparsers = plots_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc plots CMD --help` to display command-specific help.",
-    )
-
-    fix_subparsers(plots_subparsers)
-
-    SHOW_HELP = "Generate plots from metrics files."
-    plots_show_parser = plots_subparsers.add_parser(
-        "show",
-        parents=[parent_parser],
-        description=append_doc_link(SHOW_HELP, "plots/show"),
-        help=SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    plots_show_parser.add_argument(
-        "targets",
-        nargs="*",
-        help="Files to visualize (supports any file, "
-        "even when not found as `plots` in `dvc.yaml`). "
-        "Shows all plots by default.",
-    ).complete = completion.FILE
-    _add_props_arguments(plots_show_parser)
-    _add_output_arguments(plots_show_parser)
-    plots_show_parser.set_defaults(func=CmdPlotsShow)
-
-    PLOTS_DIFF_HELP = (
-        "Show multiple versions of plot metrics "
-        "by plotting them in a single image."
-    )
-    plots_diff_parser = plots_subparsers.add_parser(
-        "diff",
-        parents=[parent_parser],
-        description=append_doc_link(PLOTS_DIFF_HELP, "plots/diff"),
-        help=PLOTS_DIFF_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    plots_diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help=(
-            "Specific plots file(s) to visualize "
-            "(even if not found as `plots` in `dvc.yaml`). "
-            "Shows all tracked plots by default."
-        ),
-        metavar="<paths>",
-    ).complete = completion.FILE
-    plots_diff_parser.add_argument(
-        "-e",
-        "--experiment",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
-    plots_diff_parser.add_argument(
-        "revisions", nargs="*", default=None, help="Git commits to plot from",
-    )
-    _add_props_arguments(plots_diff_parser)
-    _add_output_arguments(plots_diff_parser)
-    plots_diff_parser.set_defaults(func=CmdPlotsDiff)
-
-    PLOTS_MODIFY_HELP = "Modify display properties of plot metrics files."
-    plots_modify_parser = plots_subparsers.add_parser(
-        "modify",
-        parents=[parent_parser],
-        description=append_doc_link(PLOTS_MODIFY_HELP, "plots/modify"),
-        help=PLOTS_MODIFY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    plots_modify_parser.add_argument(
-        "target", help="Metric file to set properties to",
-    ).complete = completion.FILE
-    _add_props_arguments(plots_modify_parser)
-    plots_modify_parser.add_argument(
-        "--unset",
-        nargs="*",
-        metavar="<property>",
-        help="Unset one or more display properties.",
-    )
-    plots_modify_parser.set_defaults(func=CmdPlotsModify)
-
-
-def _add_props_arguments(parser):
-    parser.add_argument(
-        "-t",
-        "--template",
-        nargs="?",
-        default=None,
-        help=(
-            "Special JSON or HTML schema file to inject with the data. "
-            "See {}".format(
-                format_link("https://man.dvc.org/plots#plot-templates")
-            )
-        ),
-        metavar="<path>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-x", default=None, help="Field name for X axis.", metavar="<field>"
-    )
-    parser.add_argument(
-        "-y", default=None, help="Field name for Y axis.", metavar="<field>"
-    )
-    parser.add_argument(
-        "--no-header",
-        action="store_false",
-        dest="header",
-        default=None,  # Use default None to distinguish when it's not used
-        help="Provided CSV or TSV datafile does not have a header.",
-    )
-    parser.add_argument(
-        "--title", default=None, metavar="<text>", help="Plot title."
-    )
-    parser.add_argument(
-        "--x-label", default=None, help="X axis label", metavar="<text>"
-    )
-    parser.add_argument(
-        "--y-label", default=None, help="Y axis label", metavar="<text>"
-    )
-
-
-def _add_output_arguments(parser):
-    parser.add_argument(
-        "-o",
-        "--out",
-        default=None,
-        help="Destination path to save plots to",
-        metavar="<path>",
-    ).complete = completion.DIR
-    parser.add_argument(
-        "--show-vega",
-        action="store_true",
-        default=False,
-        help="Show output in Vega format.",
-    )

dvc/command/remote.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.config import ConfigError
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 from dvc.utils import format_link
@@ -16,18 +17,18 @@ class CmdRemote(CmdConfig):
             self.args.name = self.args.name.lower()
 
     def _check_exists(self, conf):
-        from dvc.config import ConfigError
-
         if self.args.name not in conf["remote"]:
-            raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
+            raise ConfigError(
+                "remote '{}' doesn't exists.".format(self.args.name)
+            )
 
 
 class CmdRemoteAdd(CmdRemote):
     def run(self):
-        from dvc.config import ConfigError
-
         if self.args.default:
-            logger.info(f"Setting '{self.args.name}' as a default remote.")
+            logger.info(
+                "Setting '{}' as a default remote.".format(self.args.name)
+            )
 
         with self.config.edit(self.args.level) as conf:
             if self.args.name in conf["remote"] and not self.args.force:
@@ -49,14 +50,13 @@ class CmdRemoteRemove(CmdRemote):
             self._check_exists(conf)
             del conf["remote"][self.args.name]
 
-        up_to_level = self.args.level or "repo"
         # Remove core.remote refs to this remote in any shadowing configs
         for level in reversed(self.config.LEVELS):
             with self.config.edit(level) as conf:
                 if conf["core"].get("remote") == self.args.name:
                     del conf["core"]["remote"]
 
-            if level == up_to_level:
+            if level == self.args.level:
                 break
 
         return 0
@@ -64,29 +64,17 @@ class CmdRemoteRemove(CmdRemote):
 
 class CmdRemoteModify(CmdRemote):
     def run(self):
-        from dvc.config import merge
-
         with self.config.edit(self.args.level) as conf:
-            merged = self.config.load_config_to_level(self.args.level)
-            merge(merged, conf)
-            self._check_exists(merged)
-
-            if self.args.name not in conf["remote"]:
-                conf["remote"][self.args.name] = {}
-            section = conf["remote"][self.args.name]
-            if self.args.unset:
-                section.pop(self.args.option, None)
-            else:
-                section[self.args.option] = self.args.value
+            self._check_exists(conf)
+            conf["remote"][self.args.name][self.args.option] = self.args.value
         return 0
 
 
 class CmdRemoteDefault(CmdRemote):
     def run(self):
-        from dvc.config import ConfigError
 
         if self.args.name is None and not self.args.unset:
-            conf = self.config.read(self.args.level)
+            conf = self.config.load_one(self.args.level)
             try:
                 print(conf["core"]["remote"])
             except KeyError:
@@ -97,61 +85,18 @@ class CmdRemoteDefault(CmdRemote):
                 if self.args.unset:
                     conf["core"].pop("remote", None)
                 else:
-                    merged_conf = self.config.load_config_to_level(
-                        self.args.level
-                    )
-                    if (
-                        self.args.name in conf["remote"]
-                        or self.args.name in merged_conf["remote"]
-                    ):
-                        conf["core"]["remote"] = self.args.name
-                    else:
-                        raise ConfigError(
-                            "default remote must be present in remote list."
-                        )
+                    conf["core"]["remote"] = self.args.name
         return 0
 
 
 class CmdRemoteList(CmdRemote):
     def run(self):
-        conf = self.config.read(self.args.level)
+        conf = self.config.load_one(self.args.level)
         for name, conf in conf["remote"].items():
             logger.info("{}\t{}".format(name, conf["url"]))
         return 0
 
 
-class CmdRemoteRename(CmdRemote):
-    def _rename_default(self, conf):
-        if conf["core"].get("remote") == self.args.name:
-            conf["core"]["remote"] = self.args.new
-
-    def run(self):
-        from dvc.config import ConfigError
-
-        all_config = self.config.load_config_to_level(None)
-        if self.args.new in all_config.get("remote", {}):
-            raise ConfigError(
-                "Rename failed. Remote name '{}' already exists.".format(
-                    {self.args.new}
-                )
-            )
-
-        with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
-            conf["remote"][self.args.new] = conf["remote"][self.args.name]
-            del conf["remote"][self.args.name]
-            self._rename_default(conf)
-
-        up_to_level = self.args.level or "repo"
-        for level in reversed(self.config.LEVELS):
-            if level == up_to_level:
-                break
-            with self.config.edit(level) as level_conf:
-                self._rename_default(level_conf)
-
-        return 0
-
-
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
@@ -182,7 +127,7 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="Remote location. See full list of supported URLs at {}".format(
+        help="URL. See full list of supported urls at {}".format(
             format_link("https://man.dvc.org/remote")
         ),
     )
@@ -211,7 +156,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_default_parser.add_argument(
-        "name", nargs="?", help="Name of the remote"
+        "name", nargs="?", help="Name of the remote."
     )
     remote_default_parser.add_argument(
         "-u",
@@ -230,7 +175,7 @@ def add_parser(subparsers, parent_parser):
         help=REMOTE_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remote_modify_parser.add_argument("name", help="Name of the remote")
+    remote_modify_parser.add_argument("name", help="Name of the remote.")
     remote_modify_parser.add_argument(
         "option", help="Name of the option to modify."
     )
@@ -268,14 +213,3 @@ def add_parser(subparsers, parent_parser):
         "name", help="Name of the remote to remove."
     )
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
-    REMOTE_RENAME_HELP = "Rename a DVC remote"
-    remote_rename_parser = remote_subparsers.add_parser(
-        "rename",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
-        help=REMOTE_RENAME_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    remote_rename_parser.add_argument("name", help="Remote to be renamed")
-    remote_rename_parser.add_argument("new", help="New name of the remote")
-    remote_rename_parser.set_defaults(func=CmdRemoteRename)

dvc/command/remove.py

@@ -1,29 +1,48 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+import dvc.prompt as prompt
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
+
 logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
+    def _is_outs_only(self, target):
+        if not self.args.purge:
+            return True
+
+        if self.args.force:
+            return False
+
+        msg = "Are you sure you want to remove '{}' with its outputs?".format(
+            target
+        )
+
+        if prompt.confirm(msg):
+            return False
+
+        raise DvcException(
+            "Cannot purge without a confirmation from the user."
+            " Use `-f` to force."
+        )
+
     def run(self):
         for target in self.args.targets:
             try:
-                self.repo.remove(target, outs=self.args.outs)
+                outs_only = self._is_outs_only(target)
+                self.repo.remove(target, outs_only=outs_only)
             except DvcException:
-                logger.exception("")
+                logger.exception("failed to remove '{}'".format(target))
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = (
-        "Remove stages from dvc.yaml and/or"
-        " stop tracking files or directories."
-    )
+    REMOVE_HELP = "Remove DVC-tracked files or directories."
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -31,15 +50,29 @@ def add_parser(subparsers, parent_parser):
         help=REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remove_parser.add_argument(
+    remove_parser_group = remove_parser.add_mutually_exclusive_group()
+    remove_parser_group.add_argument(
+        "-o",
         "--outs",
         action="store_true",
+        default=True,
+        help="Only remove DVC-file outputs. (Default)",
+    )
+    remove_parser_group.add_argument(
+        "-p",
+        "--purge",
+        action="store_true",
         default=False,
-        help="Remove outputs as well.",
+        help="Remove DVC-file and all its outputs.",
     )
     remove_parser.add_argument(
-        "targets",
-        nargs="+",
-        help=".dvc files or stages from dvc.yaml to remove.",
-    ).complete = completion.DVC_FILE
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force purge.",
+    )
+    remove_parser.add_argument(
+        "targets", nargs="+", help="DVC-files to remove."
+    )
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -1,56 +1,77 @@
 import argparse
 import logging
+import os
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import _show_metrics
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
+from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
+from dvc.exceptions import DvcException
+
 
 logger = logging.getLogger(__name__)
 
 
 class CmdRepro(CmdBase):
     def run(self):
-        stages = self.repo.reproduce(**self._repro_kwargs)
-        if len(stages) == 0:
-            logger.info(CmdDataStatus.UP_TO_DATE_MSG)
-        else:
-            logger.info(
-                "Use `dvc push` to send your updates to " "remote storage."
-            )
+        saved_dir = os.path.realpath(os.curdir)
+        if self.args.cwd:
+            os.chdir(self.args.cwd)
 
-        if self.args.metrics:
-            metrics = self.repo.metrics.show()
-            logger.info(_show_metrics(metrics))
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
 
-        return 0
+        ret = 0
+        for target in self.args.targets:
+            try:
+                stages = self.repo.reproduce(
+                    target,
+                    single_item=self.args.single_item,
+                    force=self.args.force,
+                    dry=self.args.dry,
+                    interactive=self.args.interactive,
+                    pipeline=self.args.pipeline,
+                    all_pipelines=self.args.all_pipelines,
+                    ignore_build_cache=self.args.ignore_build_cache,
+                    no_commit=self.args.no_commit,
+                    downstream=self.args.downstream,
+                    recursive=self.args.recursive,
+                )
 
-    @property
-    def _repro_kwargs(self):
-        return {
-            "targets": self.args.targets,
-            "single_item": self.args.single_item,
-            "force": self.args.force,
-            "dry": self.args.dry,
-            "interactive": self.args.interactive,
-            "pipeline": self.args.pipeline,
-            "all_pipelines": self.args.all_pipelines,
-            "run_cache": not self.args.no_run_cache,
-            "no_commit": self.args.no_commit,
-            "downstream": self.args.downstream,
-            "recursive": self.args.recursive,
-            "force_downstream": self.args.force_downstream,
-            "pull": self.args.pull,
-            "glob": self.args.glob,
-        }
+                if len(stages) == 0:
+                    logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+
+                if self.args.metrics:
+                    metrics = self.repo.metrics.show()
+                    show_metrics(metrics)
+            except DvcException:
+                logger.exception("")
+                ret = 1
+                break
+
+        os.chdir(saved_dir)
+        return ret
 
 
-def add_arguments(repro_parser):
+def add_parser(subparsers, parent_parser):
+    REPRO_HELP = (
+        "Reproduce complete or partial pipelines by executing their stages."
+    )
+    repro_parser = subparsers.add_parser(
+        "repro",
+        parents=[parent_parser],
+        description=append_doc_link(REPRO_HELP, "repro"),
+        help=REPRO_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="Stages to reproduce. 'dvc.yaml' by default.",
-    ).complete = completion.DVCFILES_AND_STAGE
+        help="DVC-file to reproduce. 'Dvcfile' by default.",
+    )
     repro_parser.add_argument(
         "-f",
         "--force",
@@ -66,6 +87,13 @@ def add_arguments(repro_parser):
         help="Reproduce only single data item without recursive dependencies "
         "check.",
     )
+    repro_parser.add_argument(
+        "-c",
+        "--cwd",
+        default=os.path.curdir,
+        help="Directory within your repo to reproduce from.",
+        metavar="<path>",
+    )
     repro_parser.add_argument(
         "-m",
         "--metrics",
@@ -92,8 +120,8 @@ def add_arguments(repro_parser):
         "--pipeline",
         action="store_true",
         default=False,
-        help="Reproduce the whole pipeline that the specified targets "
-        "belong to.",
+        help="Reproduce the whole pipeline that the specified stage file "
+        "belongs to.",
     )
     repro_parser.add_argument(
         "-P",
@@ -110,16 +138,7 @@ def add_arguments(repro_parser):
         help="Reproduce all stages in the specified directory.",
     )
     repro_parser.add_argument(
-        "--no-run-cache",
-        action="store_true",
-        default=False,
-        help=(
-            "Execute stage commands even if they have already been run with "
-            "the same command/dependencies/outputs/etc before."
-        ),
-    )
-    repro_parser.add_argument(
-        "--force-downstream",
+        "--ignore-build-cache",
         action="store_true",
         default=False,
         help="Reproduce all descendants of a changed stage even if their "
@@ -137,33 +156,4 @@ def add_arguments(repro_parser):
         default=False,
         help="Start from the specified stages when reproducing pipelines.",
     )
-    repro_parser.add_argument(
-        "--pull",
-        action="store_true",
-        default=False,
-        help=(
-            "Try automatically pulling missing cache for outputs restored "
-            "from the run-cache."
-        ),
-    )
-    repro_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Allows targets containing shell-style wildcards.",
-    )
-
-
-def add_parser(subparsers, parent_parser):
-    REPRO_HELP = (
-        "Reproduce complete or partial pipelines by executing their stages."
-    )
-    repro_parser = subparsers.add_parser(
-        "repro",
-        parents=[parent_parser],
-        description=append_doc_link(REPRO_HELP, "repro"),
-        help=REPRO_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    add_arguments(repro_parser)
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBaseNoRepo
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)

dvc/command/run.py

@@ -1,15 +1,18 @@
 import argparse
 import logging
 
-from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.stage import parse_cmd
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
+
 logger = logging.getLogger(__name__)
 
 
 class CmdRun(CmdBase):
     def run(self):
+        overwrite = self.args.yes or self.args.overwrite_dvcfile
+
         if not any(
             [
                 self.args.deps,
@@ -17,65 +20,69 @@ class CmdRun(CmdBase):
                 self.args.outs_no_cache,
                 self.args.metrics,
                 self.args.metrics_no_cache,
-                self.args.plots,
-                self.args.plots_no_cache,
                 self.args.outs_persist,
                 self.args.outs_persist_no_cache,
-                self.args.checkpoints,
                 self.args.params,
-                self.args.cmd,
+                self.args.command,
             ]
         ):  # pragma: no cover
             logger.error(
                 "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
-                "`-m`, `-M`, `-p`, `--plots`, `--plots-no-cache`, "
-                "`--outs-persist`, `--outs-persist-no-cache`, `command`."
+                "`-m`, `-M`, `-p`, `--outs-persist`, "
+                "`--outs-persist-no-cache`, `command`."
             )
             return 1
 
         try:
             self.repo.run(
-                cmd=parse_cmd(self.args.cmd),
+                cmd=self._parsed_cmd(),
                 outs=self.args.outs,
                 outs_no_cache=self.args.outs_no_cache,
                 metrics=self.args.metrics,
                 metrics_no_cache=self.args.metrics_no_cache,
-                plots=self.args.plots,
-                plots_no_cache=self.args.plots_no_cache,
-                live=self.args.live,
-                live_no_cache=self.args.live_no_cache,
-                live_no_summary=self.args.live_no_summary,
-                live_no_html=self.args.live_no_html,
                 deps=self.args.deps,
                 params=self.args.params,
                 fname=self.args.file,
                 wdir=self.args.wdir,
-                no_exec=(self.args.no_exec or bool(self.args.checkpoints)),
-                force=self.args.force,
-                run_cache=not self.args.no_run_cache,
+                no_exec=self.args.no_exec,
+                overwrite=overwrite,
+                ignore_build_cache=self.args.ignore_build_cache,
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
                 outs_persist_no_cache=self.args.outs_persist_no_cache,
-                checkpoints=self.args.checkpoints,
                 always_changed=self.args.always_changed,
                 name=self.args.name,
-                single_stage=self.args.single_stage,
-                external=self.args.external,
-                desc=self.args.desc,
             )
         except DvcException:
-            logger.exception("")
+            logger.exception("failed to run command")
             return 1
 
         return 0
 
+    def _parsed_cmd(self):
+        """
+        We need to take into account two cases:
+
+        - ['python code.py foo bar']: Used mainly with dvc as a library
+        - ['echo', 'foo bar']: List of arguments received from the CLI
+
+        The second case would need quoting, as it was passed through:
+                dvc run echo "foo bar"
+        """
+        if len(self.args.command) < 2:
+            return " ".join(self.args.command)
+
+        return " ".join(self._quote_argument(arg) for arg in self.args.command)
+
+    def _quote_argument(self, argument):
+        if " " not in argument or '"' in argument:
+            return argument
+
+        return '"{}"'.format(argument)
+
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.stage import _add_common_args
-
-    RUN_HELP = (
-        "Generate a dvc.yaml file from a command and execute the command."
-    )
+    RUN_HELP = "Generate a stage file from a command and execute the command."
     run_parser = subparsers.add_parser(
         "run",
         parents=[parent_parser],
@@ -84,25 +91,93 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     run_parser.add_argument(
-        "-n", "--name", help="Stage name.",
+        "-d",
+        "--deps",
+        action="append",
+        default=[],
+        help="Declare dependencies for reproducible cmd.",
+        metavar="<path>",
+    )
+    run_parser.add_argument("-n", "--name", help=argparse.SUPPRESS)
+    run_parser.add_argument(
+        "-o",
+        "--outs",
+        action="append",
+        default=[],
+        help="Declare output file or directory.",
+        metavar="<filename>",
     )
     run_parser.add_argument(
-        "--file", metavar="<filename>", help=argparse.SUPPRESS,
+        "-O",
+        "--outs-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory "
+        "(do not put into DVC cache).",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "-p",
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter to use as additional dependency.",
+        metavar="[<filename>:]<params_list>",
+    )
+    run_parser.add_argument(
+        "-m",
+        "--metrics",
+        action="append",
+        default=[],
+        help="Declare output metric file or directory.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-M",
+        "--metrics-no-cache",
+        action="append",
+        default=[],
+        help="Declare output metric file or directory "
+        "(do not put into DVC cache).",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-f",
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "-w",
+        "--wdir",
+        help="Directory within your repo to run your command in.",
+        metavar="<path>",
     )
     run_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create dvc.yaml without actually running it.",
+        help="Only create stage file without actually running it.",
     )
     run_parser.add_argument(
-        "--no-run-cache",
+        "-y",
+        "--yes",
         action="store_true",
         default=False,
-        help=(
-            "Execute the command even if this stage has already been run "
-            "with the same command/dependencies/outputs/etc before."
-        ),
+        help="Deprecated, use --overwrite-dvcfile instead",
+    )
+    run_parser.add_argument(
+        "--overwrite-dvcfile",
+        action="store_true",
+        default=False,
+        help="Overwrite existing DVC-file without asking for confirmation.",
+    )
+    run_parser.add_argument(
+        "--ignore-build-cache",
+        action="store_true",
+        default=False,
+        help="Run this stage even if it has been already ran with the same "
+        "command/dependencies/outputs/etc before.",
     )
     run_parser.add_argument(
         "--no-commit",
@@ -111,10 +186,28 @@ def add_parser(subparsers, parent_parser):
         help="Don't put files/directories into cache.",
     )
     run_parser.add_argument(
-        "--single-stage",
+        "--outs-persist",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro.",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "--outs-persist-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro (do not put into DVC cache).",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "--always-changed",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help="Always consider this DVC-file as changed.",
+    )
+    run_parser.add_argument(
+        "command", nargs=argparse.REMAINDER, help="Command to execute."
     )
-    _add_common_args(run_parser)
     run_parser.set_defaults(func=CmdRun)

dvc/command/stage.py

@@ -1,364 +0,0 @@
-import argparse
-import logging
-from itertools import chain, filterfalse
-from typing import TYPE_CHECKING, Dict, Iterable, List
-
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.utils.cli_parse import parse_params
-from dvc.utils.humanize import truncate_text
-
-if TYPE_CHECKING:
-    from dvc.output.base import BaseOutput
-    from dvc.stage import Stage
-
-logger = logging.getLogger(__name__)
-
-MAX_TEXT_LENGTH = 80
-
-
-def generate_description(stage: "Stage") -> str:
-    def part_desc(outs: Iterable["BaseOutput"]) -> str:
-        return ", ".join(out.def_path for out in outs)
-
-    if not stage.deps and not stage.outs:
-        return "No outputs or dependencies"
-
-    if not stage.outs and stage.deps:
-        return "Depends on " + part_desc(stage.deps)
-
-    def is_plot_or_metric(out: "BaseOutput"):
-        return bool(out.plot) or bool(out.metric)
-
-    desc: List[str] = []
-
-    outs = list(filterfalse(is_plot_or_metric, stage.outs))
-    if outs:
-        desc.append("Outputs " + part_desc(outs))
-
-    plots_and_metrics = list(filter(is_plot_or_metric, stage.outs))
-    if plots_and_metrics:
-        desc.append("Reports " + part_desc(plots_and_metrics))
-
-    return "; ".join(desc)
-
-
-def prepare_description(
-    stage: "Stage", max_length: int = MAX_TEXT_LENGTH
-) -> str:
-    desc = stage.short_description() or generate_description(stage)
-    return truncate_text(desc, max_length)
-
-
-def prepare_stages_data(
-    stages: Iterable["Stage"],
-    description: bool = True,
-    max_length: int = MAX_TEXT_LENGTH,
-) -> Dict[str, str]:
-    return {
-        stage.addressing: prepare_description(stage, max_length=max_length)
-        if description
-        else ""
-        for stage in stages
-    }
-
-
-class CmdStageList(CmdBase):
-    def _get_stages(self) -> Iterable["Stage"]:
-        if self.args.all:
-            stages: List["Stage"] = self.repo.stages  # type: ignore
-            logger.trace(  # type: ignore[attr-defined]
-                "%d no. of stages found", len(stages)
-            )
-            return stages
-
-        # removing duplicates while maintaining order
-        collected = chain.from_iterable(
-            self.repo.stage.collect(
-                target=target,
-                recursive=self.args.recursive,
-                accept_group=True,
-            )
-            for target in self.args.targets
-        )
-        return dict.fromkeys(collected).keys()
-
-    def run(self):
-        from dvc.utils.diff import table
-
-        def log_error(relpath: str, exc: Exception):
-            if self.args.fail:
-                raise exc
-            logger.debug("Stages from %s failed to load", relpath)
-
-        # silence stage collection error by default
-        self.repo.stage_collection_error_handler = log_error
-
-        stages = self._get_stages()
-        names_only = self.args.names_only
-
-        data = prepare_stages_data(stages, description=not names_only)
-        if data:
-            print(table(header=(), rows=data.items()))
-
-        return 0
-
-
-def parse_cmd(commands: List[str]) -> str:
-    """
-    We need to take into account two cases:
-
-    - ['python code.py foo bar']: Used mainly with dvc as a library
-    - ['echo', 'foo bar']: List of arguments received from the CLI
-
-    The second case would need quoting, as it was passed through:
-            dvc run echo "foo bar"
-    """
-
-    def quote_argument(arg: str):
-        should_quote = " " in arg and '"' not in arg
-        return f'"{arg}"' if should_quote else arg
-
-    if len(commands) < 2:
-        return " ".join(commands)
-    return " ".join(map(quote_argument, commands))
-
-
-class CmdStageAdd(CmdBase):
-    def run(self):
-        kwargs = vars(self.args)
-        kwargs.update(
-            {
-                "cmd": parse_cmd(kwargs.pop("cmd")),
-                "params": parse_params(self.args.params),
-            }
-        )
-        self.repo.stage.add(**kwargs)
-        return 0
-
-
-def _add_common_args(parser):
-    parser.add_argument(
-        "-d",
-        "--deps",
-        action="append",
-        default=[],
-        help="Declare dependencies for reproducible cmd.",
-        metavar="<path>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-o",
-        "--outs",
-        action="append",
-        default=[],
-        help="Declare output file or directory.",
-        metavar="<filename>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-O",
-        "--outs-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory "
-        "(do not put into DVC cache).",
-        metavar="<filename>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-p",
-        "--params",
-        action="append",
-        default=[],
-        help="Declare parameter to use as additional dependency.",
-        metavar="[<filename>:]<params_list>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-m",
-        "--metrics",
-        action="append",
-        default=[],
-        help="Declare output metrics file.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "-M",
-        "--metrics-no-cache",
-        action="append",
-        default=[],
-        help="Declare output metrics file (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--plots",
-        action="append",
-        default=[],
-        help="Declare output plot file.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--plots-no-cache",
-        action="append",
-        default=[],
-        help="Declare output plot file (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--live", help="Declare output as dvclive.", metavar="<path>",
-    )
-    parser.add_argument(
-        "--live-no-cache",
-        help="Declare output as dvclive (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--live-no-summary",
-        action="store_true",
-        default=False,
-        help="Signal dvclive logger to not dump latest metrics file.",
-    )
-    parser.add_argument(
-        "--live-no-html",
-        action="store_true",
-        default=False,
-        help="Signal dvclive logger to not produce training report.",
-    )
-    parser.add_argument(
-        "-w",
-        "--wdir",
-        help="Directory within your repo to run your command in.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite existing stage",
-    )
-    parser.add_argument(
-        "--outs-persist",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro.",
-        metavar="<filename>",
-    )
-    parser.add_argument(
-        "--outs-persist-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro (do not put into DVC cache).",
-        metavar="<filename>",
-    )
-    parser.add_argument(
-        "-c",
-        "--checkpoints",
-        action="append",
-        default=[],
-        help=argparse.SUPPRESS,
-        metavar="<filename>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "--always-changed",
-        action="store_true",
-        default=False,
-        help="Always consider this DVC-file as changed.",
-    )
-    parser.add_argument(
-        "--external",
-        action="store_true",
-        default=False,
-        help="Allow outputs that are outside of the DVC repository.",
-    )
-    parser.add_argument(
-        "--desc",
-        type=str,
-        metavar="<text>",
-        help=(
-            "User description of the stage (optional). "
-            "This doesn't affect any DVC operations."
-        ),
-    )
-    parser.add_argument(
-        "cmd",
-        nargs=argparse.REMAINDER,
-        help="Command to execute.",
-        metavar="command",
-    )
-
-
-def add_parser(subparsers, parent_parser):
-    STAGES_HELP = "Commands to list and create stages."
-
-    stage_parser = subparsers.add_parser(
-        "stage",
-        parents=[parent_parser],
-        description=append_doc_link(STAGES_HELP, "stage"),
-        help=STAGES_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    stage_subparsers = stage_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc stage CMD --help` to display command-specific help.",
-    )
-
-    fix_subparsers(stage_subparsers)
-
-    STAGE_ADD_HELP = "Create stage"
-    stage_add_parser = stage_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(STAGE_ADD_HELP, "stage/add"),
-        help=STAGE_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    stage_add_parser.add_argument(
-        "-n", "--name", help="Name of the stage to add", required=True
-    )
-    _add_common_args(stage_add_parser)
-    stage_add_parser.set_defaults(func=CmdStageAdd)
-
-    STAGE_LIST_HELP = "List stages."
-    stage_list_parser = stage_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(STAGE_LIST_HELP, "stage/list"),
-        help=STAGE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    stage_list_parser.add_argument(
-        "targets",
-        nargs="*",
-        default=["dvc.yaml"],
-        help=(
-            "Show stages from a dvc.yaml/.dvc file or a directory. "
-            "'dvc.yaml' by default"
-        ),
-    )
-    stage_list_parser.add_argument(
-        "--all",
-        action="store_true",
-        default=False,
-        help="List all of the stages in the repo.",
-    )
-    stage_list_parser.add_argument(
-        "--fail",
-        action="store_true",
-        default=False,
-        help="Fail immediately, do not suppress any syntax errors.",
-    )
-    stage_list_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="List all stages inside the specified directory.",
-    )
-    stage_list_parser.add_argument(
-        "--names-only",
-        action="store_true",
-        default=False,
-        help="List only stage names.",
-    )
-    stage_list_parser.set_defaults(func=CmdStageList)

dvc/command/status.py

@@ -1,8 +1,7 @@
 import logging
 
 from dvc.command.data_sync import CmdDataBase
-from dvc.exceptions import DvcException
-from dvc.utils import format_link
+
 
 logger = logging.getLogger(__name__)
 
@@ -11,11 +10,6 @@ class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 20
     STATUS_INDENT = "\t"
     UP_TO_DATE_MSG = "Data and pipelines are up to date."
-    IN_SYNC_MSG = "Cache and remote '{remote}' are in sync."
-    EMPTY_PROJECT_MSG = (
-        "There are no data or pipelines tracked in this project yet.\n"
-        "See {link} to get started!"
-    ).format(link=format_link("https://dvc.org/doc/start"))
 
     def _normalize(self, s):
         s += ":"
@@ -26,7 +20,7 @@ class CmdDataStatus(CmdDataBase):
         ind = indent * self.STATUS_INDENT
 
         if isinstance(status, str):
-            logger.info(f"{ind}{status}")
+            logger.info("{}{}".format(ind, status))
             return
 
         if isinstance(status, list):
@@ -40,7 +34,7 @@ class CmdDataStatus(CmdDataBase):
             if isinstance(value, str):
                 logger.info("{}{}{}".format(ind, self._normalize(value), key))
             elif value:
-                logger.info(f"{ind}{key}:")
+                logger.info("{}{}:".format(ind, key))
                 self._show(value, indent + 1)
 
     def run(self):
@@ -55,29 +49,16 @@ class CmdDataStatus(CmdDataBase):
                 all_tags=self.args.all_tags,
                 all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
-                recursive=self.args.recursive,
             )
-
-            if self.args.quiet:
-                return bool(st)
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(st))
-            elif st:
-                self._show(st, indent)
-            elif not self.repo.stages:
-                logger.info(self.EMPTY_PROJECT_MSG)
-            elif self.args.cloud or self.args.remote:
-                remote = self.args.remote or self.repo.config["core"].get(
-                    "remote"
-                )
-                logger.info(self.IN_SYNC_MSG.format(remote=remote))
+            if st:
+                if self.args.quiet:
+                    return 1
+                else:
+                    self._show(st, indent)
             else:
                 logger.info(self.UP_TO_DATE_MSG)
 
-        except DvcException:
-            logger.exception("")
+        except Exception:
+            logger.exception("failed to obtain data status")
             return 1
         return 0

dvc/command/unprotect.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -14,7 +15,7 @@ class CmdUnprotect(CmdBase):
             try:
                 self.repo.unprotect(target)
             except DvcException:
-                msg = f"failed to unprotect '{target}'"
+                msg = "failed to unprotect '{}'".format(target)
                 logger.exception(msg)
                 return 1
         return 0
@@ -33,6 +34,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect.",
-    ).complete = completion.FILE
+        "targets", nargs="+", help="Data files/directories to unprotect."
+    )
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -1,28 +1,23 @@
 import argparse
 import logging
 
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.base import append_doc_link
+from dvc.command.base import CmdBase
 from dvc.exceptions import DvcException
 
+
 logger = logging.getLogger(__name__)
 
 
 class CmdUpdate(CmdBase):
     def run(self):
         ret = 0
-        try:
-            self.repo.update(
-                targets=self.args.targets,
-                rev=self.args.rev,
-                recursive=self.args.recursive,
-                to_remote=self.args.to_remote,
-                remote=self.args.remote,
-                jobs=self.args.jobs,
-            )
-        except DvcException:
-            logger.exception("failed update data")
-            ret = 1
+        for target in self.args.targets:
+            try:
+                self.repo.update(target, self.args.rev)
+            except DvcException:
+                logger.exception("failed to update '{}'.".format(target))
+                ret = 1
         return ret
 
 
@@ -36,42 +31,12 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help=".dvc files to update.",
-    ).complete = completion.DVC_FILE
+        "targets", nargs="+", help="DVC-files to update."
+    )
     update_parser.add_argument(
         "--rev",
         nargs="?",
         help="Git revision (e.g. SHA, branch, tag)",
         metavar="<commit>",
     )
-    update_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Update all stages in the specified directory.",
-    )
-    update_parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Update data directly on the remote",
-    )
-    update_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to perform updates to",
-        metavar="<name>",
-    )
-    update_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
     update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -1,19 +1,145 @@
 import argparse
+import itertools
 import logging
+import os
+import platform
+import uuid
+import pathlib
 
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+from dvc.utils import is_binary, relpath
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.version import __version__
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.scm.base import SCMError
+from dvc.system import System
+from dvc.utils.pkg import PKG
 
 logger = logging.getLogger(__name__)
 
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
-        from dvc.info import get_dvc_info
+        from dvc.repo import Repo
 
-        dvc_info = get_dvc_info()
-        logger.info(dvc_info)
+        info = [
+            "DVC version: {}".format(__version__),
+            "Python version: {}".format(platform.python_version()),
+            "Platform: {}".format(platform.platform()),
+            "Binary: {}".format(is_binary()),
+            "Package: {}".format(PKG),
+            "Supported remotes: {}".format(self.get_supported_remotes()),
+        ]
+
+        try:
+            repo = Repo()
+            root_directory = repo.root_dir
+
+            # cache_dir might not exist yet (e.g. after `dvc init`), and we
+            # can't auto-create it, as it might cause issues if the user
+            # later decides to enable shared cache mode with
+            # `dvc config cache.shared group`.
+            if os.path.exists(repo.cache.local.cache_dir):
+                info.append(
+                    "Cache: {}".format(self.get_linktype_support_info(repo))
+                )
+                if psutil:
+                    fs_type = self.get_fs_type(repo.cache.local.cache_dir)
+                    info.append(
+                        "Filesystem type (cache directory): {}".format(fs_type)
+                    )
+            else:
+                logger.warning(
+                    "Unable to detect supported link types, as cache "
+                    "directory '{}' doesn't exist. It is usually auto-created "
+                    "by commands such as `dvc add/fetch/pull/run/import`, "
+                    "but you could create it manually to enable this "
+                    "check.".format(relpath(repo.cache.local.cache_dir))
+                )
+
+        except NotDvcRepoError:
+            root_directory = os.getcwd()
+        except SCMError:
+            root_directory = os.getcwd()
+            info.append("Repo: dvc, git (broken)")
+        else:
+            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
+
+        if psutil:
+            fs_root = self.get_fs_type(os.path.abspath(root_directory))
+            info.append("Filesystem type (workspace): {}".format(fs_root))
+
+        logger.info("\n".join(info))
         return 0
 
+    @staticmethod
+    def get_fs_type(path):
+        partition = {
+            pathlib.Path(part.mountpoint): (part.fstype, part.device)
+            for part in psutil.disk_partitions(all=True)
+        }
+
+        path = pathlib.Path(path)
+
+        for parent in itertools.chain([path], path.parents):
+            if parent in partition:
+                return partition[parent]
+        return ("unknown", "none")
+
+    @staticmethod
+    def get_linktype_support_info(repo):
+        links = {
+            "reflink": (System.reflink, None),
+            "hardlink": (System.hardlink, System.is_hardlink),
+            "symlink": (System.symlink, System.is_symlink),
+        }
+
+        fname = "." + str(uuid.uuid4())
+        src = os.path.join(repo.cache.local.cache_dir, fname)
+        open(src, "w").close()
+        dst = os.path.join(repo.root_dir, fname)
+
+        cache = []
+
+        for name, (link, is_link) in links.items():
+            try:
+                link(src, dst)
+                status = "supported"
+                if is_link and not is_link(dst):
+                    status = "broken"
+                os.unlink(dst)
+            except DvcException:
+                status = "not supported"
+            cache.append("{name} - {status}".format(name=name, status=status))
+        os.remove(src)
+
+        return ", ".join(cache)
+
+    @staticmethod
+    def get_supported_remotes():
+        from dvc.remote import REMOTES
+
+        supported_remotes = []
+        for remote in REMOTES:
+            if not remote.get_missing_deps():
+                supported_remotes.append(remote.scheme)
+
+        return ", ".join(supported_remotes)
+
+
+def _get_dvc_repo_info(repo):
+    if repo.config.get("core", {}).get("no_scm", False):
+        return "dvc (no_scm)"
+
+    if repo.root_dir != repo.scm.root_dir:
+        return "dvc (subdir), git"
+
+    return "dvc, git"
+
 
 def add_parser(subparsers, parent_parser):
     VERSION_HELP = (
@@ -25,6 +151,5 @@ def add_parser(subparsers, parent_parser):
         description=append_doc_link(VERSION_HELP, "version"),
         help=VERSION_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        aliases=["doctor"],
     )
     version_parser.set_defaults(func=CmdVersion)

dvc/compat.py

@@ -0,0 +1,42 @@
+"""Python version compatibility code"""
+
+# Backport os.fspath() from Python 3.6
+try:
+    from os import fspath  # noqa: F821
+
+    fspath_py35 = lambda s: s  # noqa: E731
+except ImportError:
+
+    def fspath(path):
+        """Return the path representation of a path-like object.
+
+        If str or bytes is passed in, it is returned unchanged. Otherwise the
+        os.PathLike interface is used to get the path representation. If the
+        path representation is not str or bytes, TypeError is raised. If the
+        provided path is not str, bytes, or os.PathLike, TypeError is raised.
+        """
+        if isinstance(path, (str, bytes)):
+            return path
+
+        # Work from the object's type to match method resolution of other magic
+        # methods.
+        path_type = type(path)
+        try:
+            path_repr = path_type.__fspath__(path)
+        except AttributeError:
+            if hasattr(path_type, "__fspath__"):
+                raise
+            else:
+                raise TypeError(
+                    "expected str, bytes or os.PathLike object, "
+                    "not " + path_type.__name__
+                )
+        if isinstance(path_repr, (str, bytes)):
+            return path_repr
+        else:
+            raise TypeError(
+                "expected {}.__fspath__() to return str or bytes, "
+                "not {}".format(path_type.__name__, type(path_repr).__name__)
+            )
+
+    fspath_py35 = fspath

dvc/config.py

@@ -4,11 +4,25 @@ import os
 import re
 from contextlib import contextmanager
 from functools import partial
+from urllib.parse import urlparse
 
-from funcy import cached_property, compact, memoize, re_find
+import configobj
+from funcy import cached_property, compact, re_find, walk_values
+from voluptuous import (
+    ALLOW_EXTRA,
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
 
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.path_info import PathInfo
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -17,26 +31,180 @@ class ConfigError(DvcException):
     """DVC config exception."""
 
     def __init__(self, msg):
-        super().__init__(f"config file error: {msg}")
+        super().__init__("config file error: {}".format(msg))
 
 
 class NoRemoteError(ConfigError):
     pass
 
 
-@memoize
-def get_compiled_schema():
-    from voluptuous import Schema
+def supported_cache_type(types):
+    """Checks if link type config option consists only of valid values.
 
-    from .config_schema import SCHEMA
+    Args:
+        types (list/string): type(s) of links that dvc should try out.
+    """
+    if types is None:
+        return None
+    if isinstance(types, str):
+        types = [typ.strip() for typ in types.split(",")]
 
-    return Schema(SCHEMA)
+    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
+    if unsupported:
+        raise Invalid(
+            "Unsupported cache type(s): {}".format(", ".join(unsupported))
+        )
+
+    return types
 
 
-def to_bool(value):
-    from .config_schema import Bool
+# Checks that value is either true or false and converts it to bool
+to_bool = Bool = All(
+    Lower,
+    Any("true", "false"),
+    lambda v: v == "true",
+    msg="expected true or false",
+)
 
-    return Bool(value)
+
+def Choices(*choices):
+    """Checks that value belongs to the specified set of values
+
+    Args:
+        *choices: pass allowed values as arguments, or pass a list or
+            tuple as a single argument
+    """
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
+
+
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    Optional("no_traverse"): Bool,  # obsoleted
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+}
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    Optional("listobjects", default=False): Bool,
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_email": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_p12_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+}
+COMPILED_SCHEMA = Schema(SCHEMA)
 
 
 class Config(dict):
@@ -62,11 +230,7 @@ class Config(dict):
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
-    def __init__(
-        self, dvc_dir=None, validate=True, fs=None, config=None,
-    ):  # pylint: disable=super-init-not-called
-        from dvc.fs.local import LocalFileSystem
-
+    def __init__(self, dvc_dir=None, validate=True):
         self.dvc_dir = dvc_dir
 
         if not dvc_dir:
@@ -79,14 +243,11 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.wfs = LocalFileSystem(None, {"url": self.dvc_dir})
-        self.fs = fs or self.wfs
-
-        self.load(validate=validate, config=config)
+        self.load(validate=validate)
 
     @classmethod
     def get_dir(cls, level):
-        from appdirs import site_config_dir, user_config_dir
+        from appdirs import user_config_dir, site_config_dir
 
         assert level in ("global", "system")
 
@@ -122,16 +283,13 @@ class Config(dict):
         open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def load(self, validate=True, config=None):
+    def load(self, validate=True):
         """Loads config from all the config files.
 
         Raises:
             ConfigError: thrown if config has an invalid format.
         """
-        conf = self.load_config_to_level()
-
-        if config is not None:
-            merge(conf, config)
+        conf = self._load_config_to_level()
 
         if validate:
             conf = self.validate(conf)
@@ -143,45 +301,12 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
-    def _get_fs(self, level):
-        # NOTE: this might be a Gitfs, which doesn't see things outside of
-        # the repo.
-        return self.fs if level == "repo" else self.wfs
-
-    def _load_config(self, level):
-        from configobj import ConfigObj
-
-        filename = self.files[level]
-        fs = self._get_fs(level)
-
-        if fs.exists(filename, use_dvcignore=False):
-            with fs.open(filename) as fobj:
-                conf_obj = ConfigObj(fobj)
-        else:
-            conf_obj = ConfigObj()
-        return _parse_remotes(_lower_keys(conf_obj.dict()))
-
-    def _save_config(self, level, conf_dict):
-        from configobj import ConfigObj
-
-        filename = self.files[level]
-        fs = self._get_fs(level)
-
-        logger.debug(f"Writing '{filename}'.")
-
-        fs.makedirs(os.path.dirname(filename))
-
-        config = ConfigObj(_pack_remotes(conf_dict))
-        with fs.open(filename, "wb") as fobj:
-            config.write(fobj)
-        config.filename = filename
-
     def load_one(self, level):
-        conf = self._load_config(level)
+        conf = _load_config(self.files[level])
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
-        for key in get_compiled_schema().schema:
+        for key in COMPILED_SCHEMA.schema:
             conf.setdefault(key, {})
 
         return conf
@@ -191,26 +316,14 @@ class Config(dict):
         abs_conf_dir = os.path.abspath(os.path.dirname(filename))
 
         def resolve(path):
-            from .config_schema import RelPath
-
             if os.path.isabs(path) or re.match(r"\w+://", path):
                 return path
-
-            # on windows convert slashes to backslashes
-            # to have path compatible with abs_conf_dir
-            if os.path.sep == "\\" and "/" in path:
-                path = path.replace("/", "\\")
-
             return RelPath(os.path.join(abs_conf_dir, path))
 
         return Config._map_dirs(conf, resolve)
 
     @staticmethod
     def _to_relpath(conf_dir, path):
-        from dvc.utils import relpath
-
-        from .config_schema import RelPath
-
         if re.match(r"\w+://", path):
             return path
 
@@ -228,43 +341,25 @@ class Config(dict):
 
     @staticmethod
     def _map_dirs(conf, func):
-        from voluptuous import ALLOW_EXTRA, Schema
-
         dirs_schema = {
             "cache": {"dir": func},
             "remote": {
-                str: {
-                    "url": func,
-                    "gdrive_user_credentials_file": func,
-                    "gdrive_service_account_json_file_path": func,
-                    "credentialpath": func,
-                    "keyfile": func,
-                    "cert_path": func,
-                    "key_path": func,
-                }
+                str: {"url": func, "gdrive_user_credentials_file": func}
             },
         }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
 
-    def load_config_to_level(self, level=None):
+    def _load_config_to_level(self, level=None):
         merged_conf = {}
         for merge_level in self.LEVELS:
             if merge_level == level:
                 break
             if merge_level in self.files:
-                merge(merged_conf, self.load_one(merge_level))
+                _merge(merged_conf, self.load_one(merge_level))
         return merged_conf
 
-    def read(self, level=None):
-        # NOTE: we read from a merged config by default, same as git config
-        if level is None:
-            return self.load_config_to_level()
-        return self.load_one(level)
-
     @contextmanager
-    def edit(self, level=None, validate=True):
-        # NOTE: we write to repo config by default, same as git config
-        level = level or "repo"
+    def edit(self, level="repo"):
         if level in {"repo", "local"} and self.dvc_dir is None:
             raise ConfigError("Not inside a DVC repo")
 
@@ -273,25 +368,35 @@ class Config(dict):
 
         conf = self._save_paths(conf, self.files[level])
 
-        merged_conf = self.load_config_to_level(level)
-        merge(merged_conf, conf)
+        merged_conf = self._load_config_to_level(level)
+        _merge(merged_conf, conf)
+        self.validate(merged_conf)
 
-        if validate:
-            self.validate(merged_conf)
-
-        self._save_config(level, conf)
-        self.load(validate=validate)
+        _save_config(self.files[level], conf)
+        self.load()
 
     @staticmethod
     def validate(data):
-        from voluptuous import Invalid
-
         try:
-            return get_compiled_schema()(data)
+            return COMPILED_SCHEMA(data)
         except Invalid as exc:
             raise ConfigError(str(exc)) from None
 
 
+def _load_config(filename):
+    conf_obj = configobj.ConfigObj(filename)
+    return _parse_remotes(_lower_keys(conf_obj.dict()))
+
+
+def _save_config(filename, conf_dict):
+    logger.debug("Writing '{}'.".format(filename))
+    os.makedirs(os.path.dirname(filename), exist_ok=True)
+
+    config = configobj.ConfigObj(_pack_remotes(conf_dict))
+    config.filename = filename
+    config.write()
+
+
 def _parse_remotes(conf):
     result = {"remote": {}}
 
@@ -311,17 +416,17 @@ def _pack_remotes(conf):
 
     # Transform remote.name -> 'remote "name"'
     for name, val in conf["remote"].items():
-        result[f'remote "{name}"'] = val
+        result['remote "{}"'.format(name)] = val
     result.pop("remote", None)
 
     return result
 
 
-def merge(into, update):
+def _merge(into, update):
     """Merges second dict into first recursively"""
     for key, val in update.items():
         if isinstance(into.get(key), dict) and isinstance(val, dict):
-            merge(into[key], val)
+            _merge(into[key], val)
         else:
             into[key] = val
 

dvc/config_schema.py

@@ -1,225 +0,0 @@
-import os
-from urllib.parse import urlparse
-
-from funcy import walk_values
-from voluptuous import (
-    All,
-    Any,
-    Coerce,
-    Invalid,
-    Lower,
-    Optional,
-    Range,
-    Schema,
-)
-
-Bool = All(
-    Lower,
-    Any("true", "false"),
-    lambda v: v == "true",
-    msg="expected true or false",
-)
-
-
-def supported_cache_type(types):
-    """Checks if link type config option consists only of valid values.
-
-    Args:
-        types (list/string): type(s) of links that dvc should try out.
-    """
-    if types is None:
-        return None
-    if isinstance(types, str):
-        types = [typ.strip() for typ in types.split(",")]
-
-    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
-    if unsupported:
-        raise Invalid(
-            "Unsupported cache type(s): {}".format(", ".join(unsupported))
-        )
-
-    return types
-
-
-def Choices(*choices):
-    """Checks that value belongs to the specified set of values
-
-    Args:
-        *choices: pass allowed values as arguments, or pass a list or
-            tuple as a single argument
-    """
-    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
-
-
-def ByUrl(mapping):
-    schemas = walk_values(Schema, mapping)
-
-    def validate(data):
-        if "url" not in data:
-            raise Invalid("expected 'url'")
-
-        parsed = urlparse(data["url"])
-        # Windows absolute paths should really have scheme == "" (local)
-        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
-            return schemas[""](data)
-        if parsed.scheme not in schemas:
-            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
-
-        return schemas[parsed.scheme](data)
-
-    return validate
-
-
-class RelPath(str):
-    pass
-
-
-REMOTE_COMMON = {
-    "url": str,
-    "checksum_jobs": All(Coerce(int), Range(1)),
-    "jobs": All(Coerce(int), Range(1)),
-    Optional("no_traverse"): Bool,  # obsoleted
-    "verify": Bool,
-}
-LOCAL_COMMON = {
-    "type": supported_cache_type,
-    Optional("protected", default=False): Bool,  # obsoleted
-    "shared": All(Lower, Choices("group")),
-    Optional("slow_link_warning", default=True): Bool,
-}
-HTTP_COMMON = {
-    "auth": All(Lower, Choices("basic", "digest", "custom")),
-    "custom_auth_header": str,
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "ssl_verify": Bool,
-    "method": str,
-}
-WEBDAV_COMMON = {
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "token": str,
-    "cert_path": str,
-    "key_path": str,
-    "timeout": Coerce(int),
-}
-
-SCHEMA = {
-    "core": {
-        "remote": Lower,
-        "checksum_jobs": All(Coerce(int), Range(1)),
-        Optional("interactive", default=False): Bool,
-        Optional("analytics", default=True): Bool,
-        Optional("hardlink_lock", default=False): Bool,
-        Optional("no_scm", default=False): Bool,
-        Optional("autostage", default=False): Bool,
-        Optional("experiments"): Bool,  # obsoleted
-        Optional("check_update", default=True): Bool,
-    },
-    "cache": {
-        "local": str,
-        "s3": str,
-        "gs": str,
-        "hdfs": str,
-        "webhdfs": str,
-        "ssh": str,
-        "azure": str,
-        # This is for default local cache
-        "dir": str,
-        **LOCAL_COMMON,
-    },
-    "remote": {
-        str: ByUrl(
-            {
-                "": {**LOCAL_COMMON, **REMOTE_COMMON},
-                "s3": {
-                    "region": str,
-                    "profile": str,
-                    "credentialpath": str,
-                    "configpath": str,
-                    "endpointurl": str,
-                    "access_key_id": str,
-                    "secret_access_key": str,
-                    "session_token": str,
-                    Optional("listobjects", default=False): Bool,  # obsoleted
-                    Optional("use_ssl", default=True): Bool,
-                    "sse": str,
-                    "sse_kms_key_id": str,
-                    "acl": str,
-                    "grant_read": str,
-                    "grant_read_acp": str,
-                    "grant_write_acp": str,
-                    "grant_full_control": str,
-                    **REMOTE_COMMON,
-                },
-                "gs": {
-                    "projectname": str,
-                    "credentialpath": str,
-                    **REMOTE_COMMON,
-                },
-                "ssh": {
-                    "type": supported_cache_type,
-                    "port": Coerce(int),
-                    "user": str,
-                    "password": str,
-                    "ask_password": Bool,
-                    "keyfile": str,
-                    "timeout": Coerce(int),
-                    "gss_auth": Bool,
-                    "allow_agent": Bool,
-                    **REMOTE_COMMON,
-                },
-                "hdfs": {"user": str, **REMOTE_COMMON},
-                "webhdfs": {
-                    "hdfscli_config": str,
-                    "webhdfs_token": str,
-                    "user": str,
-                    "webhdfs_alias": str,
-                    **REMOTE_COMMON,
-                },
-                "azure": {
-                    "connection_string": str,
-                    "sas_token": str,
-                    "account_name": str,
-                    "account_key": str,
-                    "tenant_id": str,
-                    "client_id": str,
-                    "client_secret": str,
-                    **REMOTE_COMMON,
-                },
-                "oss": {
-                    "oss_key_id": str,
-                    "oss_key_secret": str,
-                    "oss_endpoint": str,
-                    **REMOTE_COMMON,
-                },
-                "gdrive": {
-                    "gdrive_use_service_account": Bool,
-                    "gdrive_client_id": str,
-                    "gdrive_client_secret": str,
-                    "gdrive_user_credentials_file": str,
-                    "gdrive_service_account_user_email": str,
-                    "gdrive_service_account_json_file_path": str,
-                    Optional("gdrive_trash_only", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "http": {**HTTP_COMMON, **REMOTE_COMMON},
-                "https": {**HTTP_COMMON, **REMOTE_COMMON},
-                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "remote": {str: object},  # Any of the above options are valid
-            }
-        )
-    },
-    "state": {
-        "row_limit": All(Coerce(int), Range(1)),  # obsoleted
-        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
-    },
-    # section for experimental features
-    "feature": {
-        # enabled by default. It's of no use, kept for backward compatibility.
-        Optional("parametrization", default=True): Bool
-    },
-}

dvc/daemon.py

@@ -3,12 +3,13 @@
 import inspect
 import logging
 import os
-import platform
 import sys
 from subprocess import Popen
 
 from dvc.env import DVC_DAEMON
-from dvc.utils import fix_env, is_binary
+from dvc.utils import fix_env
+from dvc.utils import is_binary
+
 
 logger = logging.getLogger(__name__)
 
@@ -16,30 +17,25 @@ CREATE_NEW_PROCESS_GROUP = 0x00000200
 DETACHED_PROCESS = 0x00000008
 
 
-def _popen(cmd, **kwargs):
-    prefix = [sys.executable]
-    if not is_binary():
-        prefix += [sys.argv[0]]
-
-    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
-
-
 def _spawn_windows(cmd, env):
-    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
+    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
 
     creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    _popen(
-        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
+    Popen(
+        cmd,
+        env=env,
+        close_fds=True,
+        shell=False,
+        creationflags=creationflags,
+        startupinfo=startupinfo,
     ).communicate()
 
 
 def _spawn_posix(cmd, env):
-    from dvc.main import main
-
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
@@ -52,6 +48,7 @@ def _spawn_posix(cmd, env):
         os._exit(1)  # pylint: disable=protected-access
 
     os.setsid()
+    os.umask(0)
 
     try:
         pid = os.fork()
@@ -65,19 +62,13 @@ def _spawn_posix(cmd, env):
     sys.stdout.close()
     sys.stderr.close()
 
-    if platform.system() == "Darwin":
-        # workaround for MacOS bug
-        # https://github.com/iterative/dvc/issues/4294
-        _popen(cmd, env=env).communicate()
-    else:
-        os.environ.update(env)
-        main(cmd)
+    Popen(cmd, env=env, close_fds=True, shell=False).communicate()
 
     os._exit(0)  # pylint: disable=protected-access
 
 
 def _spawn(cmd, env):
-    logger.debug(f"Trying to spawn '{cmd}'")
+    logger.debug("Trying to spawn '{}'".format(cmd))
 
     if os.name == "nt":
         _spawn_windows(cmd, env)
@@ -86,7 +77,7 @@ def _spawn(cmd, env):
     else:
         raise NotImplementedError
 
-    logger.debug(f"Spawned '{cmd}'")
+    logger.debug("Spawned '{}'".format(cmd))
 
 
 def daemon(args):
@@ -99,7 +90,10 @@ def daemon(args):
         logger.debug("skipping launching a new daemon.")
         return
 
-    cmd = ["daemon", "-q"] + args
+    cmd = [sys.executable]
+    if not is_binary():
+        cmd += [sys.argv[0]]
+    cmd += ["daemon", "-q"] + args
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])

dvc/dagascii.py

@@ -6,9 +6,12 @@ import os
 import pydoc
 import sys
 
-from grandalf.graphs import Edge, Graph, Vertex
+from grandalf.graphs import Edge
+from grandalf.graphs import Graph
+from grandalf.graphs import Vertex
 from grandalf.layouts import SugiyamaLayout
-from grandalf.routing import EdgeViewer, route_with_lines
+from grandalf.routing import EdgeViewer
+from grandalf.routing import route_with_lines
 
 from dvc.env import DVC_PAGER
 from dvc.utils import format_link
@@ -37,7 +40,7 @@ def find_pager():
     if env_pager:
         return make_pager(env_pager)
 
-    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
+    if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
         return make_pager(DEFAULT_PAGER_FORMATTED)
 
     logger.warning(
@@ -49,7 +52,7 @@ def find_pager():
     return pydoc.plainpager
 
 
-class VertexViewer:
+class VertexViewer(object):
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.
 
@@ -75,7 +78,7 @@ class VertexViewer:
         return self._w
 
 
-class AsciiCanvas:
+class AsciiCanvas(object):
     """Class for drawing in ASCII.
 
     Args:
@@ -96,9 +99,10 @@ class AsciiCanvas:
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
+        pager = find_pager()
         lines = map("".join, self.canvas)
         joined_lines = os.linesep.join(lines)
-        return joined_lines
+        pager(joined_lines)
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.
@@ -144,21 +148,21 @@ class AsciiCanvas:
                 if dx == 0:
                     y = y0
                 else:
-                    y = y0 + int(round((x - x0) * dy / float(dx)))
+                    y = y0 + int(round((x - x0) * dy / float((dx))))
                 self.point(x, y, char)
         elif y0 < y1:
             for y in range(y0, y1 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x0 + int(round((y - y0) * dx / float(dy)))
+                    x = x0 + int(round((y - y0) * dx / float((dy))))
                 self.point(x, y, char)
         else:
             for y in range(y1, y0 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x1 + int(round((y - y1) * dx / float(dy)))
+                    x = x1 + int(round((y - y1) * dx / float((dy))))
                 self.point(x, y, char)
 
     def text(self, x, y, text):
@@ -212,7 +216,7 @@ def _build_sugiyama_layout(vertexes, edges):
     # Y
     #
 
-    vertexes = {v: Vertex(f" {v} ") for v in vertexes}
+    vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
     # NOTE: reverting edges to correctly orientate the graph
     edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
     vertexes = vertexes.values()
@@ -222,7 +226,7 @@ def _build_sugiyama_layout(vertexes, edges):
         vertex.view = VertexViewer(vertex.data)
 
     # NOTE: determine min box length to create the best layout
-    minw = min(v.view.w for v in vertexes)
+    minw = min([v.view.w for v in vertexes])
 
     for edge in edges:
         edge.view = EdgeViewer()
@@ -315,4 +319,4 @@ def draw(vertexes, edges):
             int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
         )
 
-    return canvas.draw()
+    canvas.draw()

dvc/data_cloud.py

@@ -2,10 +2,14 @@
 
 import logging
 
+from dvc.config import NoRemoteError
+from dvc.remote import Remote
+
+
 logger = logging.getLogger(__name__)
 
 
-class DataCloud:
+class DataCloud(object):
     """Class that manages dvc remotes.
 
     Args:
@@ -20,8 +24,6 @@ class DataCloud:
         self.repo = repo
 
     def get_remote(self, name=None, command="<command>"):
-        from dvc.config import NoRemoteError
-
         if not name:
             name = self.repo.config["core"].get("remote")
 
@@ -44,13 +46,9 @@ class DataCloud:
         raise NoRemoteError(error_msg)
 
     def _init_remote(self, name):
-        from dvc.remote import get_remote
+        return Remote(self.repo, name=name)
 
-        return get_remote(self.repo, name=name)
-
-    def push(
-        self, cache, jobs=None, remote=None, show_checksums=False,
-    ):
+    def push(self, cache, jobs=None, remote=None, show_checksums=False):
         """Push data items in a cloud-agnostic way.
 
         Args:
@@ -61,18 +59,14 @@ class DataCloud:
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        remote = self.get_remote(remote, "push")
-
-        return remote.push(
-            self.repo.odb.local,
+        return self.repo.cache.local.push(
             cache,
             jobs=jobs,
+            remote=self.get_remote(remote, "push"),
             show_checksums=show_checksums,
         )
 
-    def pull(
-        self, cache, jobs=None, remote=None, show_checksums=False,
-    ):
+    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
 
         Args:
@@ -84,40 +78,26 @@ class DataCloud:
                 information messages.
         """
         remote = self.get_remote(remote, "pull")
-
-        return remote.pull(
-            self.repo.odb.local,
-            cache,
-            jobs=jobs,
-            show_checksums=show_checksums,
+        downloaded_items_num = self.repo.cache.local.pull(
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
-    def transfer(self, source, jobs=None, remote=None, command=None):
-        """Transfer data items in a cloud-agnostic way.
+        if not remote.verify:
+            self._save_pulled_checksums(cache)
 
-        Args:
-            source (str): url for the source location.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to compare
-                cache to. By default remote from core.remote config option
-                is used.
-            command (str): the command which is benefitting from this function
-                (to be used for reporting better error messages).
-        """
-        from dvc.fs import get_cloud_fs
+        return downloaded_items_num
 
-        from_fs = get_cloud_fs(self.repo, url=source)
-        remote = self.get_remote(remote, command)
-        return remote.transfer(from_fs, from_fs.path_info, jobs=jobs)
+    def _save_pulled_checksums(self, cache):
+        for checksum in cache.scheme_keys("local"):
+            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
+            if self.repo.cache.local.exists(cache_file):
+                # We can safely save here, as existing corrupted files will
+                # be removed upon status, while files corrupted during
+                # download will not be moved from tmp_file
+                # (see `RemoteBASE.download()`)
+                self.repo.state.save(cache_file, checksum)
 
-    def status(
-        self,
-        cache,
-        jobs=None,
-        remote=None,
-        show_checksums=False,
-        log_missing=True,
-    ):
+    def status(self, cache, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
@@ -128,18 +108,8 @@ class DataCloud:
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
-            log_missing (bool): log warning messages if file doesn't exist
-                neither in cache, neither in cloud.
         """
         remote = self.get_remote(remote, "status")
-        return remote.status(
-            self.repo.odb.local,
-            cache,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            log_missing=log_missing,
+        return self.repo.cache.local.status(
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
-
-    def get_url_for(self, remote, checksum):
-        remote = self.get_remote(remote)
-        return str(remote.odb.hash_to_path_info(checksum))

dvc/dependency/__init__.py

@@ -1,48 +1,49 @@
-from collections import defaultdict
 from urllib.parse import urlparse
+from collections import defaultdict
 
 import dvc.output as output
-from dvc.dependency.azure import AzureDependency
 from dvc.dependency.gs import GSDependency
 from dvc.dependency.hdfs import HDFSDependency
 from dvc.dependency.http import HTTPDependency
 from dvc.dependency.https import HTTPSDependency
 from dvc.dependency.local import LocalDependency
-from dvc.dependency.param import ParamsDependency
 from dvc.dependency.s3 import S3Dependency
 from dvc.dependency.ssh import SSHDependency
-from dvc.dependency.webdav import WebDAVDependency
-from dvc.dependency.webdavs import WebDAVSDependency
-from dvc.dependency.webhdfs import WebHDFSDependency
+from dvc.dependency.param import ParamsDependency
 from dvc.output.base import BaseOutput
+from dvc.remote import Remote
 from dvc.scheme import Schemes
-
-from ..fs import get_cloud_fs
 from .repo import RepoDependency
 
+
+DEPS = [
+    GSDependency,
+    HDFSDependency,
+    HTTPDependency,
+    HTTPSDependency,
+    S3Dependency,
+    SSHDependency,
+    # NOTE: LocalDependency is the default choice
+]
+
 DEP_MAP = {
     Schemes.LOCAL: LocalDependency,
     Schemes.SSH: SSHDependency,
     Schemes.S3: S3Dependency,
-    Schemes.AZURE: AzureDependency,
     Schemes.GS: GSDependency,
     Schemes.HDFS: HDFSDependency,
     Schemes.HTTP: HTTPDependency,
     Schemes.HTTPS: HTTPSDependency,
-    Schemes.WEBDAV: WebDAVDependency,
-    Schemes.WEBDAVS: WebDAVSDependency,
-    Schemes.WEBHDFS: WebHDFSDependency,
 }
 
 
 # NOTE: schema for dependencies is basically the same as for outputs, but
 # without output-specific entries like 'cache' (whether or not output is
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
-# output is a metrics file and how to parse it, see `-M` flag for `dvc run`).
+# output is a metric file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
 del SCHEMA[BaseOutput.PARAM_CACHE]
 del SCHEMA[BaseOutput.PARAM_METRIC]
-del SCHEMA[BaseOutput.PARAM_DESC]
 SCHEMA.update(RepoDependency.REPO_SCHEMA)
 SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 
@@ -50,8 +51,8 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 def _get(stage, p, info):
     parsed = urlparse(p) if p else None
     if parsed and parsed.scheme == "remote":
-        fs = get_cloud_fs(stage.repo, name=parsed.netloc)
-        return DEP_MAP[fs.scheme](stage, p, info, fs=fs)
+        remote = Remote(stage.repo, name=parsed.netloc)
+        return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
 
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)
@@ -61,8 +62,10 @@ def _get(stage, p, info):
         params = info.pop(ParamsDependency.PARAM_PARAMS)
         return ParamsDependency(stage, p, params)
 
-    dep_cls = DEP_MAP.get(parsed.scheme, LocalDependency)
-    return dep_cls(stage, p, info)
+    for d in DEPS:
+        if d.supported(p):
+            return d(stage, p, info)
+    return LocalDependency(stage, p, info)
 
 
 def loadd_from(stage, d_list):
@@ -74,34 +77,33 @@ def loadd_from(stage, d_list):
 
 
 def loads_from(stage, s_list, erepo=None):
-    assert isinstance(s_list, list)
-    info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
-    return [_get(stage, s, info.copy()) for s in s_list]
+    ret = []
+    for s in s_list:
+        info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
+        ret.append(_get(stage, s, info))
+    return ret
 
 
-def _merge_params(s_list):
-    d = defaultdict(list)
-    default_file = ParamsDependency.DEFAULT_PARAMS_FILE
-    for key in s_list:
-        if isinstance(key, str):
-            d[default_file].append(key)
-            continue
-        if not isinstance(key, dict):
-            msg = "Only list of str/dict is supported. Got: "
-            msg += f"'{type(key).__name__}'."
-            raise ValueError(msg)
-
-        for k, params in key.items():
-            if not isinstance(params, list):
-                msg = "Expected list of params for custom params file "
-                msg += f"'{k}', got '{type(params).__name__}'."
-                raise ValueError(msg)
-            d[k].extend(params)
-    return d
+def _parse_params(path_params):
+    path, _, params_str = path_params.rpartition(":")
+    params = params_str.split(",")
+    return path, params
 
 
 def loads_params(stage, s_list):
-    d = _merge_params(s_list)
-    return [
-        ParamsDependency(stage, path, params) for path, params in d.items()
-    ]
+    # Creates an object for each unique file that is referenced in the list
+    params_by_path = defaultdict(list)
+    for s in s_list:
+        path, params = _parse_params(s)
+        params_by_path[path].extend(params)
+
+    d_list = []
+    for path, params in params_by_path.items():
+        d_list.append(
+            {
+                BaseOutput.PARAM_PATH: path,
+                ParamsDependency.PARAM_PARAMS: params,
+            }
+        )
+
+    return loadd_from(stage, d_list)

dvc/dependency/azure.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.azure import AzureFileSystem
-
-
-class AzureDependency(BaseDependency, BaseOutput):
-    FS_CLS = AzureFileSystem

dvc/dependency/base.py

@@ -1,33 +1,31 @@
-from typing import Type
-
 from dvc.exceptions import DvcException
 
 
 class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = f"dependency '{path}' does not exist"
+        msg = "dependency '{}' does not exist".format(path)
         super().__init__(msg)
 
 
 class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = f"dependency '{path}' is not a file or directory"
+        msg = "dependency '{}' is not a file or directory".format(path)
         super().__init__(msg)
 
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__(f"DVC file '{path}' cannot be a dependency.")
+        super().__init__(
+            "Stage file '{}' cannot be a dependency.".format(path)
+        )
 
 
-class BaseDependency:
+class BaseDependency(object):
     IS_DEPENDENCY = True
 
-    DoesNotExistError = DependencyDoesNotExistError  # type: Type[DvcException]
-    IsNotFileOrDirError = (
-        DependencyIsNotFileOrDirError
-    )  # type: Type[DvcException]
-    IsStageFileError = DependencyIsStageFileError  # type: Type[DvcException]
+    DoesNotExistError = DependencyDoesNotExistError
+    IsNotFileOrDirError = DependencyIsNotFileOrDirError
+    IsStageFileError = DependencyIsStageFileError
 
     def update(self, rev=None):
         pass

dvc/dependency/gs.py

@@ -1,8 +1,6 @@
 from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.gs import GSFileSystem
+from dvc.output.gs import GSOutput
 
 
-class GSDependency(BaseDependency, BaseOutput):
-    FS_CLS = GSFileSystem
+class GSDependency(BaseDependency, GSOutput):
+    pass

dvc/dependency/http.py

@@ -1,8 +1,7 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
-
-from ..fs.http import HTTPFileSystem
+from dvc.remote.http import HTTPRemote
 
 
 class HTTPDependency(BaseDependency, BaseOutput):
-    FS_CLS = HTTPFileSystem
+    REMOTE = HTTPRemote

dvc/dependency/https.py

@@ -1,6 +1,6 @@
-from ..fs.https import HTTPSFileSystem
 from .http import HTTPDependency
+from dvc.remote.https import HTTPSRemote
 
 
 class HTTPSDependency(HTTPDependency):
-    FS_CLS = HTTPSFileSystem
+    REMOTE = HTTPSRemote

dvc/dependency/param.py

@@ -1,13 +1,13 @@
 import os
+import yaml
 from collections import defaultdict
 
 import dpath.util
 from voluptuous import Any
 
+from dvc.compat import fspath_py35
 from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
-from dvc.hash_info import HashInfo
-from dvc.utils.serialize import LOADERS, ParseError
 
 
 class MissingParamsError(DvcException):
@@ -32,7 +32,7 @@ class ParamsDependency(LocalDependency):
             else:
                 assert isinstance(params, dict)
                 self.params = list(params.keys())
-                info = {self.PARAM_PARAMS: params}
+                info = params
 
         super().__init__(
             stage,
@@ -41,61 +41,50 @@ class ParamsDependency(LocalDependency):
             info=info,
         )
 
-    def dumpd(self):
-        ret = super().dumpd()
-        if not self.hash_info:
-            ret[self.PARAM_PARAMS] = self.params
-        return ret
+    def save(self):
+        super().save()
+        self.info = self.save_info()
 
-    def fill_values(self, values=None):
-        """Load params values dynamically."""
-        if not values:
-            return
-        info = {}
-        for param in self.params:
-            if param in values:
-                info[param] = values[param]
-        self.hash_info = HashInfo(self.PARAM_PARAMS, info)
+    def status(self):
+        status = super().status()
 
-    def workspace_status(self):
-        status = super().workspace_status()
-
-        if status.get(str(self)) == "deleted":
+        if status[str(self)] == "deleted":
             return status
 
         status = defaultdict(dict)
-        info = self.hash_info.value if self.hash_info else {}
-        actual = self.read_params()
+        info = self.read_params()
         for param in self.params:
-            if param not in actual.keys():
+            if param not in info.keys():
                 st = "deleted"
-            elif param not in info:
+            elif param not in self.info:
                 st = "new"
-            elif actual[param] != info[param]:
+            elif info[param] != self.info[param]:
                 st = "modified"
             else:
-                assert actual[param] == info[param]
+                assert info[param] == self.info[param]
                 continue
 
             status[str(self)][param] = st
 
         return status
 
-    def status(self):
-        return self.workspace_status()
+    def dumpd(self):
+        return {
+            self.PARAM_PATH: self.def_path,
+            self.PARAM_PARAMS: self.info or self.params,
+        }
 
     def read_params(self):
         if not self.exists:
             return {}
 
-        suffix = self.path_info.suffix.lower()
-        loader = LOADERS[suffix]
-        try:
-            config = loader(self.path_info, fs=self.repo.fs)
-        except ParseError as exc:
-            raise BadParamFileError(
-                f"Unable to read parameters from '{self}'"
-            ) from exc
+        with self.repo.tree.open(fspath_py35(self.path_info), "r") as fobj:
+            try:
+                config = yaml.safe_load(fobj)
+            except yaml.YAMLError as exc:
+                raise BadParamFileError(
+                    "Unable to read parameters from '{}'".format(self)
+                ) from exc
 
         ret = {}
         for param in self.params:
@@ -105,7 +94,7 @@ class ParamsDependency(LocalDependency):
                 pass
         return ret
 
-    def get_hash(self):
+    def save_info(self):
         info = self.read_params()
 
         missing_params = set(self.params) - set(info.keys())
@@ -116,4 +105,4 @@ class ParamsDependency(LocalDependency):
                 )
             )
 
-        return HashInfo(self.PARAM_PARAMS, info)
+        return info

dvc/dependency/repo.py

@@ -1,8 +1,10 @@
+import os
+
 from voluptuous import Required
 
-from dvc.path_info import PathInfo
-
 from .local import LocalDependency
+from dvc.exceptions import OutputNotFoundError
+from dvc.path_info import PathInfo
 
 
 class RepoDependency(LocalDependency):
@@ -23,7 +25,7 @@ class RepoDependency(LocalDependency):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, fs, path):
+    def _parse_path(self, remote, path):
         return None
 
     @property
@@ -39,76 +41,50 @@ class RepoDependency(LocalDependency):
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    def _make_repo(self, *, locked=True, **kwargs):
+    def _make_repo(self, *, locked=True):
         from dvc.external_repo import external_repo
 
         d = self.def_repo
         rev = (d.get("rev_lock") if locked else None) or d.get("rev")
-        return external_repo(d["url"], rev=rev, **kwargs)
-
-    def _get_hash(self, locked=True):
-        from dvc.objects.stage import stage
+        return external_repo(d["url"], rev=rev)
 
+    def _get_checksum(self, locked=True):
         with self._make_repo(locked=locked) as repo:
-            path_info = PathInfo(repo.root_dir) / self.def_path
-            return stage(
-                self.repo.odb.local,
-                path_info,
-                repo.repo_fs,
-                follow_subrepos=False,
-            ).hash_info
+            try:
+                return repo.find_out_by_relpath(self.def_path).info["md5"]
+            except OutputNotFoundError:
+                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+                # We are polluting our repo cache with some dir listing here
+                return self.repo.cache.local.get_checksum(path)
 
-    def workspace_status(self):
-        current = self._get_hash(locked=True)
-        updated = self._get_hash(locked=False)
+    def status(self):
+        current_checksum = self._get_checksum(locked=True)
+        updated_checksum = self._get_checksum(locked=False)
 
-        if current != updated:
+        if current_checksum != updated_checksum:
             return {str(self): "update available"}
 
         return {}
 
-    def status(self):
-        return self.workspace_status()
-
     def save(self):
         pass
 
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def download(self, to, jobs=None):
-        from dvc.checkout import checkout
-        from dvc.config import NoRemoteError
-        from dvc.exceptions import NoOutputOrStageError
-        from dvc.objects import save
-        from dvc.objects.stage import stage
-
-        odb = self.repo.odb.local
-
-        with self._make_repo(cache_dir=odb.cache_dir) as repo:
+    def download(self, to):
+        with self._make_repo() as repo:
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
-            path_info = PathInfo(repo.root_dir) / self.def_path
-            try:
-                repo.fetch([path_info.fspath], jobs=jobs, recursive=True)
-            except (NoOutputOrStageError, NoRemoteError):
-                pass
-            obj = stage(
-                odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False,
-            )
-            save(odb, obj, jobs=jobs)
+                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
 
-        checkout(to.path_info, to.fs, obj, odb)
+            if hasattr(repo, "cache"):
+                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
+
+            repo.pull_to(self.def_path, to.path_info)
 
     def update(self, rev=None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
 
         with self._make_repo(locked=False) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
-
-    def changed_checksum(self):
-        # From current repo point of view what describes RepoDependency is its
-        # origin project url and rev_lock, and it makes RepoDependency
-        # immutable, hence its impossible for checksum to change.
-        return False
+            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()

dvc/dependency/webdav.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.webdav import WebDAVFileSystem
-
-
-class WebDAVDependency(BaseDependency, BaseOutput):
-    FS_CLS = WebDAVFileSystem

dvc/dependency/webdavs.py

@@ -1,6 +0,0 @@
-from ..fs.webdavs import WebDAVSFileSystem
-from .webdav import WebDAVDependency
-
-
-class WebDAVSDependency(WebDAVDependency):
-    FS_CLS = WebDAVSFileSystem

dvc/dependency/webhdfs.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.webhdfs import WebHDFSOutput
-
-
-class WebHDFSDependency(BaseDependency, WebHDFSOutput):
-    pass

dvc/dir_info.py

@@ -1,117 +0,0 @@
-import posixpath
-from operator import itemgetter
-
-from funcy import cached_property
-from pygtrie import Trie
-
-from .hash_info import HashInfo
-
-
-def _diff(ancestor, other, allow_removed=False):
-    from dictdiffer import diff
-
-    from dvc.exceptions import MergeError
-
-    allowed = ["add"]
-    if allow_removed:
-        allowed.append("remove")
-
-    result = list(diff(ancestor, other))
-    for typ, _, _ in result:
-        if typ not in allowed:
-            raise MergeError(
-                "unable to auto-merge directories with diff that contains "
-                f"'{typ}'ed files"
-            )
-    return result
-
-
-def _merge(ancestor, our, their):
-    import copy
-
-    from dictdiffer import patch
-
-    our_diff = _diff(ancestor, our)
-    if not our_diff:
-        return copy.deepcopy(their)
-
-    their_diff = _diff(ancestor, their)
-    if not their_diff:
-        return copy.deepcopy(our)
-
-    # make sure there are no conflicting files
-    _diff(our, their, allow_removed=True)
-
-    return patch(our_diff + their_diff, ancestor)
-
-
-class DirInfo:
-    PARAM_RELPATH = "relpath"
-
-    def __init__(self):
-        self._dict = {}
-
-    def as_dict(self):
-        return self._dict.copy()
-
-    def add(self, key, hash_info):
-        self.__dict__.pop("trie", None)
-        self._dict[key] = hash_info
-
-    def get(self, key, default=None):
-        return self._dict.get(key, default)
-
-    @cached_property
-    def trie(self):
-        return Trie(self._dict)
-
-    @property
-    def size(self):
-        try:
-            return sum(hash_info.size for _, hash_info in self._dict.items())
-        except TypeError:
-            return None
-
-    @property
-    def nfiles(self):
-        return len(self._dict)
-
-    def items(self):
-        for key, hash_info in self._dict.items():
-            yield key, hash_info
-
-    @classmethod
-    def from_list(cls, lst):
-        ret = DirInfo()
-        for _entry in lst:
-            entry = _entry.copy()
-            relpath = entry.pop(cls.PARAM_RELPATH)
-            parts = tuple(relpath.split(posixpath.sep))
-            hash_info = HashInfo.from_dict(entry)
-            ret.add(parts, hash_info)
-        return ret
-
-    def to_list(self):
-        # Sorting the list by path to ensure reproducibility
-        return sorted(
-            (
-                {
-                    # NOTE: not using hash_info.to_dict() because we don't want
-                    # size/nfiles fields at this point.
-                    hash_info.name: hash_info.value,
-                    self.PARAM_RELPATH: posixpath.sep.join(parts),
-                }
-                for parts, hash_info in self._dict.items()  # noqa: B301
-            ),
-            key=itemgetter(self.PARAM_RELPATH),
-        )
-
-    def merge(self, ancestor, their):
-        merged_dict = _merge(
-            ancestor.as_dict(), self.as_dict(), their.as_dict(),
-        )
-
-        merged = DirInfo()
-        for key, hi in merged_dict.items():
-            merged.add(key, hi)
-        return merged

dvc/dvcfile.py

@@ -1,52 +1,39 @@
-import collections
 import contextlib
-import logging
 import os
-from typing import TYPE_CHECKING, Any, Union
+import logging
+
+import dvc.prompt as prompt
 
 from voluptuous import MultipleInvalid
-
+from dvc import serialize
 from dvc.exceptions import DvcException
-from dvc.parsing.versions import LOCKFILE_VERSION, SCHEMA_KWD
-from dvc.stage import serialize
+from dvc.stage.loader import SingleStageLoader, StageLoader
 from dvc.stage.exceptions import (
     StageFileBadNameError,
     StageFileDoesNotExistError,
-    StageFileFormatError,
     StageFileIsNotDvcFileError,
+    StageFileFormatError,
+    StageFileAlreadyExistsError,
 )
-from dvc.types import AnyPath
 from dvc.utils import relpath
 from dvc.utils.collections import apply_diff
-from dvc.utils.serialize import (
-    dump_yaml,
-    modify_yaml,
-    parse_yaml,
-    parse_yaml_for_update,
+from dvc.utils.stage import (
+    dump_stage_file,
+    parse_stage,
+    parse_stage_for_update,
 )
 
-if TYPE_CHECKING:
-    from dvc.repo import Repo
-
 logger = logging.getLogger(__name__)
 
 DVC_FILE = "Dvcfile"
 DVC_FILE_SUFFIX = ".dvc"
-PIPELINE_FILE = "dvc.yaml"
-PIPELINE_LOCK = "dvc.lock"
-
-
-class FileIsGitIgnored(DvcException):
-    def __init__(self, path):
-        super().__init__(f"'{path}' is git-ignored.")
+PIPELINE_FILE = "pipelines.yaml"
+PIPELINE_LOCK = "pipelines.lock"
 
 
 class LockfileCorruptedError(DvcException):
-    pass
-
-
-class ParametrizedDumpError(DvcException):
-    pass
+    def __init__(self, path):
+        super().__init__("Lockfile '{}' is corrupted.".format(path))
 
 
 def is_valid_filename(path):
@@ -58,20 +45,16 @@ def is_valid_filename(path):
 
 def is_dvc_file(path):
     return os.path.isfile(path) and (
-        is_valid_filename(path) or is_lock_file(path)
+        is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
     )
 
 
-def is_lock_file(path):
-    return os.path.basename(path) == PIPELINE_LOCK
-
-
 def check_dvc_filename(path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
-            "bad DVC file name '{}'. DVC files should be named "
-            "'{}' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
-                relpath(path), PIPELINE_FILE, os.path.basename(path)
+            "bad DVC-file name '{}'. DVC-files should be named "
+            "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
+                relpath(path), os.path.basename(path)
             )
         )
 
@@ -79,10 +62,9 @@ def check_dvc_filename(path):
 class FileMixin:
     SCHEMA = None
 
-    def __init__(self, repo, path, verify=True, **kwargs):
+    def __init__(self, repo, path, **kwargs):
         self.repo = repo
         self.path = path
-        self.verify = verify
 
     def __repr__(self):
         return "{}: {}".format(
@@ -98,164 +80,133 @@ class FileMixin:
         ) == os.path.abspath(other.path)
 
     def __str__(self):
-        return f"{self.__class__.__name__}: {self.relpath}"
+        return "{}: {}".format(self.__class__.__name__, self.relpath)
 
     @property
     def relpath(self):
         return relpath(self.path)
 
     def exists(self):
-        return self.repo.fs.exists(self.path)
-
-    def _is_git_ignored(self):
-        from dvc.fs.local import LocalFileSystem
-
-        return isinstance(
-            self.repo.fs, LocalFileSystem
-        ) and self.repo.scm.is_ignored(self.path)
-
-    def _verify_filename(self):
-        if self.verify:
-            check_dvc_filename(self.path)
-
-    def _check_gitignored(self):
-        if self._is_git_ignored():
-            raise FileIsGitIgnored(self.path)
+        return self.repo.tree.exists(self.path)
 
     def _load(self):
         # it raises the proper exceptions by priority:
         # 1. when the file doesn't exists
-        # 2. filename is not a DVC file
+        # 2. filename is not a DVC-file
         # 3. path doesn't represent a regular file
-        # 4. when the file is git ignored
         if not self.exists():
-            is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
-            raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
-
-        self._verify_filename()
-        if not self.repo.fs.isfile(self.path):
+            raise StageFileDoesNotExistError(self.path)
+        check_dvc_filename(self.path)
+        if not self.repo.tree.isfile(self.path):
             raise StageFileIsNotDvcFileError(self.path)
 
-        self._check_gitignored()
-        with self.repo.fs.open(self.path, encoding="utf-8") as fd:
+        with self.repo.tree.open(self.path) as fd:
             stage_text = fd.read()
-        d = parse_yaml(stage_text, self.path)
-        return self.validate(d, self.relpath), stage_text
+        d = parse_stage(stage_text, self.path)
+        self.validate(d, self.path)
+        return d, stage_text
 
     @classmethod
     def validate(cls, d, fname=None):
-        assert isinstance(cls.SCHEMA, collections.abc.Callable)
+        assert cls.SCHEMA
         try:
-            return cls.SCHEMA(d)  # pylint: disable=not-callable
+            cls.SCHEMA(d)
         except MultipleInvalid as exc:
-            raise StageFileFormatError(f"'{fname}' format error: {exc}")
+            raise StageFileFormatError(fname, exc)
 
-    def remove(self, force=False):  # pylint: disable=unused-argument
+    def remove_with_prompt(self, force=False):
+        raise NotImplementedError
+
+    def remove(self, force=False):
         with contextlib.suppress(FileNotFoundError):
             os.unlink(self.path)
 
     def dump(self, stage, **kwargs):
         raise NotImplementedError
 
-    def merge(self, ancestor, other):
-        raise NotImplementedError
-
 
 class SingleStageFile(FileMixin):
     from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
-    from dvc.stage.loader import SingleStageLoader as LOADER
+
+    def __init__(self, repo, path):
+        super().__init__(repo, path)
 
     @property
     def stage(self):
         data, raw = self._load()
-        return self.LOADER.load_stage(self, data, raw)
+        return SingleStageLoader.load_stage(self, data, raw)
 
     @property
     def stages(self):
         data, raw = self._load()
-        return self.LOADER(self, data, raw)
+        return SingleStageLoader(self, data, raw)
 
     def dump(self, stage, **kwargs):
         """Dumps given stage appropriately in the dvcfile."""
         from dvc.stage import PipelineStage
 
         assert not isinstance(stage, PipelineStage)
-        if self.verify:
-            check_dvc_filename(self.path)
+        check_dvc_filename(self.path)
         logger.debug(
             "Saving information to '{file}'.".format(file=relpath(self.path))
         )
-        dump_yaml(self.path, serialize.to_single_stage_file(stage))
-        self.repo.scm.track_file(self.relpath)
+        dump_stage_file(self.path, serialize.to_single_stage_file(stage))
+        self.repo.scm.track_file(relpath(self.path))
+
+    def remove_with_prompt(self, force=False):
+        if not self.exists():
+            return
+
+        msg = (
+            "'{}' already exists. Do you wish to run the command and "
+            "overwrite it?".format(relpath(self.path))
+        )
+        if not (force or prompt.confirm(msg)):
+            raise StageFileAlreadyExistsError(self.path)
 
-    def remove_stage(self, stage):  # pylint: disable=unused-argument
         self.remove()
 
-    def merge(self, ancestor, other):
-        assert isinstance(ancestor, SingleStageFile)
-        assert isinstance(other, SingleStageFile)
-
-        stage = self.stage
-        stage.merge(ancestor.stage, other.stage)
-        self.dump(stage)
-
 
 class PipelineFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
 
     from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
-    from dvc.stage.loader import StageLoader as LOADER
 
     @property
     def _lockfile(self):
         return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
 
-    def dump(
-        self, stage, update_pipeline=True, update_lock=True, **kwargs
-    ):  # pylint: disable=arguments-differ
+    def dump(self, stage, update_pipeline=False, **kwargs):
         """Dumps given stage appropriately in the dvcfile."""
         from dvc.stage import PipelineStage
 
         assert isinstance(stage, PipelineStage)
-        if self.verify:
-            check_dvc_filename(self.path)
-
+        check_dvc_filename(self.path)
+        self._dump_lockfile(stage)
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)
 
-        if update_lock:
-            self._dump_lockfile(stage)
-
     def _dump_lockfile(self, stage):
         self._lockfile.dump(stage)
 
-    @staticmethod
-    def _check_if_parametrized(stage):
-        if stage.raw_data.parametrized:
-            raise ParametrizedDumpError(f"cannot dump a parametrized {stage}")
-
     def _dump_pipeline_file(self, stage):
-        self._check_if_parametrized(stage)
+        data = {}
+        if self.exists():
+            with open(self.path, "r") as fd:
+                data = parse_stage_for_update(fd.read(), self.path)
+        else:
+            open(self.path, "w+").close()
+
+        data["stages"] = data.get("stages", {})
         stage_data = serialize.to_pipeline_file(stage)
+        if data["stages"].get(stage.name):
+            orig_stage_data = data["stages"][stage.name]
+            apply_diff(stage_data[stage.name], orig_stage_data)
+        else:
+            data["stages"].update(stage_data)
 
-        with modify_yaml(self.path, fs=self.repo.fs) as data:
-            if not data:
-                logger.info("Creating '%s'", self.relpath)
-
-            data["stages"] = data.get("stages", {})
-            existing_entry = stage.name in data["stages"]
-            action = "Modifying" if existing_entry else "Adding"
-            logger.info(
-                "%s stage '%s' in '%s'", action, stage.name, self.relpath
-            )
-
-            if existing_entry:
-                orig_stage_data = data["stages"][stage.name]
-                apply_diff(stage_data[stage.name], orig_stage_data)
-            else:
-                data["stages"].update(stage_data)
-
-        self.repo.scm.track_file(self.relpath)
+        dump_stage_file(self.path, data)
+        self.repo.scm.track_file(relpath(self.path))
 
     @property
     def stage(self):
@@ -267,7 +218,7 @@ class PipelineFile(FileMixin):
     def stages(self):
         data, _ = self._load()
         lockfile_data = self._lockfile.load()
-        return self.LOADER(self, data, lockfile_data)
+        return StageLoader(self, data.get("stages", {}), lockfile_data)
 
     def remove(self, force=False):
         if not force:
@@ -277,157 +228,42 @@ class PipelineFile(FileMixin):
         super().remove()
         self._lockfile.remove()
 
-    def remove_stage(self, stage):
-        self._lockfile.remove_stage(stage)
-        if not self.exists():
-            return
-
-        with open(self.path, "r", encoding="utf-8") as f:
-            d = parse_yaml_for_update(f.read(), self.path)
-
-        self.validate(d, self.path)
-        if stage.name not in d.get("stages", {}):
-            return
-
-        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
-        del d["stages"][stage.name]
-
-        if d["stages"]:
-            dump_yaml(self.path, d)
-        else:
-            super().remove()
-
-    def merge(self, ancestor, other):
-        raise NotImplementedError
-
-
-def get_lockfile_schema(d):
-    from dvc.schema import (
-        COMPILED_LOCKFILE_V1_SCHEMA,
-        COMPILED_LOCKFILE_V2_SCHEMA,
-    )
-
-    schema = {
-        LOCKFILE_VERSION.V1: COMPILED_LOCKFILE_V1_SCHEMA,
-        LOCKFILE_VERSION.V2: COMPILED_LOCKFILE_V2_SCHEMA,
-    }
-
-    version = LOCKFILE_VERSION.from_dict(d)
-    return schema[version]
-
-
-def migrate_lock_v1_to_v2(d, version_info):
-    stages = {k: v for k, v in d.items()}
-
-    for key in stages:
-        d.pop(key)
-
-    # forcing order, meta should always be at the top
-    d.update(version_info)
-    d["stages"] = stages
-
 
 class Lockfile(FileMixin):
-    @classmethod
-    def validate(cls, d, fname=None):
-        schema = get_lockfile_schema(d)
-        try:
-            return schema(d)
-        except MultipleInvalid as exc:
-            raise StageFileFormatError(f"'{fname}' format error: {exc}")
-
-    def _verify_filename(self):
-        pass  # lockfile path is hardcoded, so no need to verify here
-
-    def _load(self):
-        try:
-            return super()._load()
-        except StageFileDoesNotExistError:
-            # we still need to account for git-ignored dvc.lock file
-            # even though it may not exist or have been .dvcignored
-            self._check_gitignored()
-            return {}, ""
-        except StageFileFormatError as exc:
-            raise LockfileCorruptedError(
-                f"Lockfile '{self.relpath}' is corrupted."
-            ) from exc
+    from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
 
     def load(self):
-        d, _ = self._load()
-        return d
-
-    @property
-    def latest_version_info(self):
-        version = LOCKFILE_VERSION.V2.value  # pylint:disable=no-member
-        return {SCHEMA_KWD: version}
+        if not self.exists():
+            return {}
+        with self.repo.tree.open(self.path) as fd:
+            data = parse_stage(fd.read(), self.path)
+        try:
+            self.validate(data, fname=self.path)
+        except StageFileFormatError:
+            raise LockfileCorruptedError(self.path)
+        return data
 
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage)
-
-        with modify_yaml(self.path, fs=self.repo.fs) as data:
-            version = LOCKFILE_VERSION.from_dict(data)
-            if version == LOCKFILE_VERSION.V1:
-                logger.info(
-                    "Migrating lock file '%s' from v1 to v2", self.relpath
-                )
-                migrate_lock_v1_to_v2(data, self.latest_version_info)
-            else:
-                if not data:
-                    data.update(self.latest_version_info)
-                    # order is important, meta should always be at the top
-                    logger.info("Generating lock file '%s'", self.relpath)
-
-            data["stages"] = data.get("stages", {})
-            modified = data["stages"].get(stage.name, {}) != stage_data.get(
-                stage.name, {}
-            )
-            if modified:
-                logger.info("Updating lock file '%s'", self.relpath)
-
-            data["stages"].update(stage_data)
-
-        if modified:
-            self.repo.scm.track_file(self.relpath)
-
-    def remove_stage(self, stage):
         if not self.exists():
-            return
-
-        with open(self.path, encoding="utf-8") as f:
-            d = parse_yaml_for_update(f.read(), self.path)
-        self.validate(d, self.path)
-
-        version = LOCKFILE_VERSION.from_dict(d)
-        data = d if version == LOCKFILE_VERSION.V1 else d.get("stages", {})
-        if stage.name not in data:
-            return
-
-        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
-        del data[stage.name]
-
-        if data:
-            dump_yaml(self.path, d)
+            data = stage_data
+            open(self.path, "w+").close()
         else:
-            self.remove()
+            with self.repo.tree.open(self.path, "r") as fd:
+                data = parse_stage_for_update(fd.read(), self.path)
+            data.update(stage_data)
 
-    def merge(self, ancestor, other):
-        raise NotImplementedError
+        dump_stage_file(self.path, data)
+        self.repo.scm.track_file(relpath(self.path))
 
 
 class Dvcfile:
-    def __new__(cls, repo: "Repo", path: AnyPath, **kwargs: Any):
+    def __new__(cls, repo, path, **kwargs):
         assert path
         assert repo
 
-        return make_dvcfile(repo, path, **kwargs)
-
-
-DVCFile = Union["PipelineFile", "SingleStageFile"]
-
-
-def make_dvcfile(repo: "Repo", path: AnyPath, **kwargs: Any) -> DVCFile:
-    _, ext = os.path.splitext(str(path))
-    if ext in [".yaml", ".yml"]:
-        return PipelineFile(repo, path, **kwargs)
-    # fallback to single stage file for better error messages
-    return SingleStageFile(repo, path, **kwargs)
+        _, ext = os.path.splitext(path)
+        if ext in [".yaml", ".yml"]:
+            return PipelineFile(repo, path, **kwargs)
+        # fallback to single stage file for better error messages
+        return SingleStageFile(repo, path, **kwargs)

dvc/env.py

@@ -1,8 +1,2 @@
-DVC_CHECKPOINT = "DVC_CHECKPOINT"
 DVC_DAEMON = "DVC_DAEMON"
 DVC_PAGER = "DVC_PAGER"
-DVC_ROOT = "DVC_ROOT"
-DVCLIVE_PATH = "DVCLIVE_PATH"
-DVCLIVE_SUMMARY = "DVCLIVE_SUMMARY"
-DVCLIVE_HTML = "DVCLIVE_HTML"
-DVCLIVE_RESUME = "DVCLIVE_RESUME"

dvc/exceptions.py

@@ -1,15 +1,11 @@
 """Exceptions raised by the dvc."""
-from typing import List
+
+from dvc.utils import relpath, format_link
 
 
 class DvcException(Exception):
     """Base class for all dvc exceptions."""
 
-    def __init__(self, msg, *args):
-        assert msg
-        self.msg = msg
-        super().__init__(msg, *args)
-
 
 class InvalidArgumentError(ValueError, DvcException):
     """Thrown if arguments are invalid."""
@@ -25,37 +21,32 @@ class OutputDuplicationError(DvcException):
     """
 
     def __init__(self, output, stages):
-        from funcy import first
-
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
-        if len(stages) == 1:
-            msg = "output '{}' is already specified in {}.".format(
-                output, first(stages)
-            )
-        else:
-            msg = "output '{}' is already specified in stages:\n{}".format(
-                output, "\n".join(f"\t- {s.addressing}" for s in stages),
-            )
+        msg = (
+            "file/directory '{}' is specified as an output in more than one "
+            "stage: \n{}\n"
+            "This is not allowed. Consider using a different output name."
+        ).format(
+            output, "\n".join("\t{}".format(s.addressing) for s in stages)
+        )
         super().__init__(msg)
         self.stages = stages
         self.output = output
 
 
 class OutputNotFoundError(DvcException):
-    """Thrown if a file/directory is not found as an output in any pipeline.
+    """Thrown if a file/directory not found in repository pipelines.
 
     Args:
         output (unicode): path to the file/directory.
     """
 
     def __init__(self, output, repo=None):
-        from dvc.utils import relpath
-
         self.output = output
         self.repo = repo
         super().__init__(
-            "Unable to find DVC file with output '{path}'".format(
+            "Unable to find DVC-file with output '{path}'".format(
                 path=relpath(self.output)
             )
         )
@@ -90,7 +81,10 @@ class CircularDependencyError(DvcException):
     def __init__(self, dependency):
         assert isinstance(dependency, str)
 
-        msg = "'{}' is specified as an output and as a dependency."
+        msg = (
+            "file/directory '{}' is specified as an output and as a "
+            "dependency."
+        )
         super().__init__(msg.format(dependency))
 
 
@@ -104,11 +98,11 @@ class ArgumentDuplicationError(DvcException):
 
     def __init__(self, path):
         assert isinstance(path, str)
-        super().__init__(f"file '{path}' is specified more than once.")
+        super().__init__("file '{}' is specified more than once.".format(path))
 
 
 class MoveNotDataSourceError(DvcException):
-    """Thrown when trying to move a file/directory that is not an output
+    """Thrown if attempted to move a file/directory that is not an output
     in a data source stage.
 
     Args:
@@ -139,8 +133,14 @@ class DvcParserError(DvcException):
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
-        msg = "Pipeline has a cycle involving: {}.".format(
-            ", ".join(s.addressing for s in stages)
+        stages = "\n".join(
+            "\t- {}".format(stage.addressing) for stage in stages
+        )
+        msg = (
+            "you've introduced a cycle in your pipeline that involves "
+            "the following stages:"
+            "\n"
+            "{stages}".format(stages=stages)
         )
         super().__init__(msg)
 
@@ -148,7 +148,7 @@ class CyclicGraphError(DvcException):
 class ConfirmRemoveError(DvcException):
     def __init__(self, path):
         super().__init__(
-            "unable to remove '{}' without a confirmation. Use "
+            "unable to remove '{}' without a confirmation from the user. Use "
             "`-f` to force.".format(path)
         )
 
@@ -160,49 +160,36 @@ class InitError(DvcException):
 class ReproductionError(DvcException):
     def __init__(self, dvc_file_name):
         self.path = dvc_file_name
-        super().__init__(f"failed to reproduce '{dvc_file_name}'")
+        super().__init__("failed to reproduce '{}'".format(dvc_file_name))
 
 
 class BadMetricError(DvcException):
     def __init__(self, paths):
         super().__init__(
             "the following metrics do not exist, "
-            "are not metrics files or are malformed: {paths}".format(
-                paths=", ".join(f"'{path}'" for path in paths)
+            "are not metric files or are malformed: {paths}".format(
+                paths=", ".join("'{}'".format(path) for path in paths)
             )
         )
 
 
-class MetricsError(DvcException):
-    pass
-
-
-class NoMetricsParsedError(MetricsError):
-    def __init__(self, command):
+class NoMetricsError(DvcException):
+    def __init__(self):
         super().__init__(
-            f"Could not parse {command} files. Use `-v` option to see more "
-            "details."
+            "no metric files in this repository. "
+            "Use `dvc metrics add` to add a metric file to track."
         )
 
 
-class NoMetricsFoundError(MetricsError):
-    def __init__(self, command, run_options):
+class StageFileCorruptedError(DvcException):
+    def __init__(self, path):
+        path = relpath(path)
         super().__init__(
-            f"No {command} files in this repository. "
-            f"Use `{run_options}` options for "
-            f"`dvc run` to mark stage outputs as {command}."
+            "unable to read DVC-file: {} "
+            "YAML file structure is corrupted".format(path)
         )
 
 
-class MetricDoesNotExistError(MetricsError):
-    def __init__(self, targets: List[str]):
-        if len(targets) == 1:
-            msg = "'{}' does not exist."
-        else:
-            msg = "'{}' do not exist."
-        super().__init__(msg.format(", ".join(targets)))
-
-
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super().__init__(
@@ -219,7 +206,7 @@ class OverlappingOutputPathsError(DvcException):
 
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target):
-        super().__init__(f"Did you mean `git checkout {target}`?")
+        super().__init__("Did you mean `git checkout {}`?".format(target))
 
 
 class ETagMismatchError(DvcException):
@@ -231,11 +218,10 @@ class ETagMismatchError(DvcException):
 
 
 class FileMissingError(DvcException):
-    def __init__(self, path, hint=None):
+    def __init__(self, path):
         self.path = path
-        hint = "" if hint is None else f". {hint}"
         super().__init__(
-            f"Can't find '{path}' neither locally nor on remote{hint}"
+            "Can't find '{}' neither locally nor on remote".format(path)
         )
 
 
@@ -249,8 +235,6 @@ class DvcIgnoreInCollectedDirError(DvcException):
 
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
-        from dvc.utils import format_link
-
         super().__init__(
             "Hook '{}' already exists. Please refer to {} for more "
             "info.".format(
@@ -263,28 +247,28 @@ class DownloadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(f"{amount} files failed to download")
+        super().__init__(
+            "{amount} files failed to download".format(amount=amount)
+        )
 
 
 class UploadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(f"{amount} files failed to upload")
+        super().__init__(
+            "{amount} files failed to upload".format(amount=amount)
+        )
 
 
 class CheckoutError(DvcException):
     def __init__(self, target_infos, stats=None):
-        from dvc.utils import error_link
-
         self.target_infos = target_infos
         self.stats = stats
         targets = [str(t) for t in target_infos]
         m = (
-            "Checkout failed for following targets:\n{}\nIs your "
-            "cache up to date?\n{}".format(
-                "\n".join(targets), error_link("missing-files"),
-            )
+            "Checkout failed for following targets:\n{}\nDid you "
+            "forget to fetch?".format("\n".join(targets))
         )
         super().__init__(m)
 
@@ -296,14 +280,12 @@ class CollectCacheError(DvcException):
 class NoRemoteInExternalRepoError(DvcException):
     def __init__(self, url):
         super().__init__(
-            f"No DVC remote is specified in target repository '{url}'."
+            "No DVC remote is specified in target repository '{}'.".format(url)
         )
 
 
 class NoOutputInExternalRepoError(DvcException):
     def __init__(self, path, external_repo_path, external_repo_url):
-        from dvc.utils import relpath
-
         super().__init__(
             "Output '{}' not found in target repository '{}'".format(
                 relpath(path, external_repo_path), external_repo_url
@@ -313,7 +295,7 @@ class NoOutputInExternalRepoError(DvcException):
 
 class HTTPError(DvcException):
     def __init__(self, code, reason):
-        super().__init__(f"'{code} {reason}'")
+        super().__init__("'{} {}'".format(code, reason))
 
 
 class PathMissingError(DvcException):
@@ -321,21 +303,23 @@ class PathMissingError(DvcException):
         "The path '{}' does not exist in the target repository '{}'"
         " neither as a DVC output nor as a Git-tracked file."
     )
-    default_msg_dvc_only = (
+    default_msg_output_only = (
         "The path '{}' does not exist in the target repository '{}'"
         " as an DVC output."
     )
 
-    def __init__(self, path, repo, dvc_only=False):
-        msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
+    def __init__(self, path, repo, output_only=False):
+        msg = (
+            self.default_msg
+            if not output_only
+            else self.default_msg_output_only
+        )
         super().__init__(msg.format(path, repo))
-        self.dvc_only = dvc_only
+        self.output_only = output_only
 
 
 class RemoteCacheRequiredError(DvcException):
     def __init__(self, path_info):
-        from dvc.utils import format_link
-
         super().__init__(
             (
                 "Current operation was unsuccessful because '{}' requires "
@@ -349,38 +333,5 @@ class RemoteCacheRequiredError(DvcException):
         )
 
 
-class IsADirectoryError(DvcException):  # noqa,pylint:disable=redefined-builtin
+class IsADirectoryError(DvcException):
     """Raised when a file operation is requested on a directory."""
-
-
-class NoOutputOrStageError(DvcException):
-    """
-    Raised when the target is neither an output nor a stage name in dvc.yaml
-    """
-
-    def __init__(self, target, file):
-        super().__init__(
-            f"'{target}' "
-            f"does not exist as an output or a stage name in '{file}'"
-        )
-
-
-class MergeError(DvcException):
-    pass
-
-
-class CacheLinkError(DvcException):
-    from dvc.utils import format_link
-
-    SUPPORT_LINK = "See {} for more information.".format(
-        format_link(
-            "https://dvc.org/doc/user-guide/troubleshooting#cache-types"
-        )
-    )
-
-    def __init__(self, path_infos):
-        msg = "No possible cache link types for '{}'. {}".format(
-            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK,
-        )
-        super().__init__(msg)
-        self.path_infos = path_infos

dvc/external_repo.py

@@ -1,111 +1,59 @@
 import logging
 import os
 import tempfile
-import threading
 from contextlib import contextmanager
-from typing import Dict
+from distutils.dir_util import copy_tree
+import threading
 
-from funcy import retry, wrap_with
+from funcy import retry, suppress, wrap_with, cached_property
 
-from dvc.exceptions import (
-    FileMissingError,
-    NoOutputInExternalRepoError,
-    NoRemoteInExternalRepoError,
-    NotDvcRepoError,
-    OutputNotFoundError,
-    PathMissingError,
-)
+from dvc.path_info import PathInfo
+from dvc.compat import fspath
 from dvc.repo import Repo
-from dvc.utils import relpath
+from dvc.config import NoRemoteError, NotDvcRepoError
+from dvc.exceptions import NoRemoteInExternalRepoError, CheckoutError
+from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
+from dvc.exceptions import FileMissingError, PathMissingError
+from dvc.utils.fs import remove, fs_copy, move
+from dvc.utils import tmp_fname
+from dvc.scm.git import Git
+
 
 logger = logging.getLogger(__name__)
 
 
 @contextmanager
-def external_repo(
-    url, rev=None, for_write=False, cache_dir=None, cache_types=None, **kwargs
-):
-    from dvc.config import NoRemoteError
-    from dvc.scm.git import Git
-
+def external_repo(url, rev=None, for_write=False):
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
-    # Local HEAD points to the tip of whatever branch we first cloned from
-    # (which may not be the default branch), use origin/HEAD here to get
-    # the tip of the default branch
-    rev = rev or "refs/remotes/origin/HEAD"
-
-    cache_config = {
-        "cache": {
-            "dir": cache_dir or _get_cache_dir(url),
-            "type": cache_types,
-        }
-    }
-
-    config = _get_remote_config(url) if os.path.isdir(url) else {}
-    config.update(cache_config)
-
-    def make_repo(path, **_kwargs):
-        _config = cache_config.copy()
-        if os.path.isdir(url):
-            rel = os.path.relpath(path, _kwargs["scm"].root_dir)
-            repo_path = os.path.join(url, rel)
-            _config.update(_get_remote_config(repo_path))
-        return Repo(path, config=_config, **_kwargs)
-
-    root_dir = path if for_write else os.path.realpath(path)
-    repo_kwargs = dict(
-        root_dir=root_dir,
-        url=url,
-        scm=None if for_write else Git(root_dir),
-        rev=None if for_write else rev,
-        config=config,
-        repo_factory=make_repo,
-        **kwargs,
-    )
-
-    if "subrepos" not in repo_kwargs:
-        repo_kwargs["subrepos"] = True
-
-    if "uninitialized" not in repo_kwargs:
-        repo_kwargs["uninitialized"] = True
-
-    repo = Repo(**repo_kwargs)
+    try:
+        repo = ExternalRepo(path, url)
+    except NotDvcRepoError:
+        repo = ExternalGitRepo(path, url)
 
     try:
         yield repo
-    except NoRemoteError as exc:
-        raise NoRemoteInExternalRepoError(url) from exc
+    except NoRemoteError:
+        raise NoRemoteInExternalRepoError(url)
     except OutputNotFoundError as exc:
         if exc.repo is repo:
-            raise NoOutputInExternalRepoError(
-                exc.output, repo.root_dir, url
-            ) from exc
+            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
         raise
     except FileMissingError as exc:
-        raise PathMissingError(exc.path, url) from exc
+        raise PathMissingError(exc.path, url)
     finally:
         repo.close()
         if for_write:
             _remove(path)
 
 
-CLONES: Dict[str, str] = {}
-CACHE_DIRS: Dict[str, str] = {}
-
-
-@wrap_with(threading.Lock())
-def _get_cache_dir(url):
-    try:
-        cache_dir = CACHE_DIRS[url]
-    except KeyError:
-        cache_dir = CACHE_DIRS[url] = tempfile.mkdtemp("dvc-cache")
-    return cache_dir
+CLONES = {}
+CACHE_DIRS = {}
 
 
 def clean_repos():
     # Outside code should not see cache while we are removing
-    paths = [path for path, _ in CLONES.values()] + list(CACHE_DIRS.values())
+    paths = list(CLONES.values()) + list(CACHE_DIRS.values())
     CLONES.clear()
     CACHE_DIRS.clear()
 
@@ -113,28 +61,142 @@ def clean_repos():
         _remove(path)
 
 
-def _get_remote_config(url):
-    try:
-        repo = Repo(url)
-    except NotDvcRepoError:
-        return {}
+class ExternalRepo(Repo):
+    def __init__(self, root_dir, url):
+        super().__init__(root_dir)
+        self.url = url
+        self._set_cache_dir()
+        self._fix_upstream()
 
-    try:
-        name = repo.config["core"].get("remote")
-        if not name:
-            # Fill the empty upstream entry with a new remote pointing to the
-            # original repo's cache location.
-            name = "auto-generated-upstream"
-            return {
-                "core": {"remote": name},
-                "remote": {name: {"url": repo.odb.local.cache_dir}},
-            }
+    def pull_to(self, path, to_info):
+        """
+        Pull the corresponding file or directory specified by `path` and
+        checkout it into `to_info`.
 
-        # Use original remote to make sure that we are using correct url,
-        # credential paths, etc if they are relative to the config location.
-        return {"remote": {name: repo.config["remote"][name]}}
-    finally:
-        repo.close()
+        It works with files tracked by Git and DVC, and also local files
+        outside the repository.
+        """
+        out = None
+        path_info = PathInfo(self.root_dir) / path
+
+        with suppress(OutputNotFoundError):
+            (out,) = self.find_outs_by_path(fspath(path_info), strict=False)
+
+        try:
+            if out and out.use_cache:
+                self._pull_cached(out, path_info, to_info)
+                return
+
+            # Check if it is handled by Git (it can't have an absolute path)
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(fspath(path_info), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    def _pull_cached(self, out, path_info, dest):
+        with self.state:
+            tmp = PathInfo(tmp_fname(dest))
+            src = tmp / path_info.relative_to(out.path_info)
+
+            out.path_info = tmp
+
+            # Only pull unless all needed cache is present
+            if out.changed_cache(filter_info=src):
+                self.cloud.pull(out.get_used_cache(filter_info=src))
+
+            try:
+                out.checkout(filter_info=src)
+            except CheckoutError:
+                raise FileNotFoundError
+
+            move(src, dest)
+            remove(tmp)
+
+    @wrap_with(threading.Lock())
+    def _set_cache_dir(self):
+        try:
+            cache_dir = CACHE_DIRS[self.url]
+        except KeyError:
+            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
+
+        self.cache.local.cache_dir = cache_dir
+
+    def _fix_upstream(self):
+        if not os.path.isdir(self.url):
+            return
+
+        try:
+            src_repo = Repo(self.url)
+        except NotDvcRepoError:
+            # If ExternalRepo does not throw NotDvcRepoError and Repo does,
+            # the self.url might be a bare git repo.
+            # NOTE: This will fail to resolve remote with relative path,
+            # same as if it was a remote DVC repo.
+            return
+
+        try:
+            remote_name = self.config["core"].get("remote")
+            if remote_name:
+                self._fix_local_remote(src_repo, remote_name)
+            else:
+                self._add_upstream(src_repo)
+        finally:
+            src_repo.close()
+
+    def _fix_local_remote(self, src_repo, remote_name):
+        # If a remote URL is relative to the source repo,
+        # it will have changed upon config load and made
+        # relative to this new repo. Restore the old one here.
+        new_remote = self.config["remote"][remote_name]
+        old_remote = src_repo.config["remote"][remote_name]
+        if new_remote["url"] != old_remote["url"]:
+            new_remote["url"] = old_remote["url"]
+
+    def _add_upstream(self, src_repo):
+        # Fill the empty upstream entry with a new remote pointing to the
+        # original repo's cache location.
+        cache_dir = src_repo.cache.local.cache_dir
+        self.config["remote"]["auto-generated-upstream"] = {"url": cache_dir}
+        self.config["core"]["remote"] = "auto-generated-upstream"
+
+
+class ExternalGitRepo:
+    def __init__(self, root_dir, url):
+        self.root_dir = root_dir
+        self.url = url
+
+    @cached_property
+    def scm(self):
+        return Git(self.root_dir)
+
+    def close(self):
+        if "scm" in self.__dict__:
+            self.scm.close()
+
+    def find_out_by_relpath(self, path):
+        raise OutputNotFoundError(path, self)
+
+    def pull_to(self, path, to_info):
+        try:
+            # Git handled files can't have absolute path
+            if os.path.isabs(path):
+                raise FileNotFoundError
+
+            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
+
+    @contextmanager
+    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        """Opens a specified resource as a file object."""
+        try:
+            abs_path = os.path.join(self.root_dir, path)
+            with open(abs_path, mode, encoding=encoding) as fd:
+                yield fd
+        except FileNotFoundError:
+            raise PathMissingError(path, self.url)
 
 
 def _cached_clone(url, rev, for_write=False):
@@ -144,14 +206,14 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    from distutils.dir_util import copy_tree
+    if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
+        return CLONES[url, rev]
 
-    # even if we have already cloned this repo, we may need to
-    # fetch/fast-forward to get specified rev
-    clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
+    clone_path = _clone_default_branch(url, rev)
+    rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
 
-    if not for_write and (url) in CLONES:
-        return CLONES[url][0]
+    if not for_write and (url, rev_sha) in CLONES:
+        return CLONES[url, rev_sha]
 
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
@@ -159,22 +221,21 @@ def _cached_clone(url, rev, for_write=False):
     copy_tree(clone_path, repo_path)
 
     # Check out the specified revision
-    if for_write:
+    if rev is not None:
         _git_checkout(repo_path, rev)
-    else:
-        CLONES[url] = (repo_path, shallow)
+
+    if not for_write:
+        CLONES[url, rev_sha] = repo_path
     return repo_path
 
 
 @wrap_with(threading.Lock())
-def _clone_default_branch(url, rev, for_write=False):
+def _clone_default_branch(url, rev):
     """Get or create a clean clone of the url.
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
-    from dvc.scm.git import Git
-
-    clone_path, shallow = CLONES.get(url, (None, False))
+    clone_path = CLONES.get(url)
 
     git = None
     try:
@@ -182,62 +243,21 @@ def _clone_default_branch(url, rev, for_write=False):
             git = Git(clone_path)
             # Do not pull for known shas, branches and tags might move
             if not Git.is_sha(rev) or not git.has_rev(rev):
-                if shallow:
-                    # If we are missing a rev in a shallow clone, fallback to
-                    # a full (unshallowed) clone. Since fetching specific rev
-                    # SHAs is only available in certain git versions, if we
-                    # have need to reference multiple specific revs for a
-                    # given repo URL it is easier/safer for us to work with
-                    # full clones in this case.
-                    logger.debug("erepo: unshallowing clone for '%s'", url)
-                    _unshallow(git)
-                    shallow = False
-                    CLONES[url] = (clone_path, shallow)
-                else:
-                    logger.debug("erepo: git pull '%s'", url)
-                    git.pull()
+                logger.debug("erepo: git pull %s", url)
+                git.pull()
         else:
-            logger.debug("erepo: git clone '%s' to a temporary dir", url)
+            logger.debug("erepo: git clone %s to a temporary dir", url)
             clone_path = tempfile.mkdtemp("dvc-clone")
-            if not for_write and rev and not Git.is_sha(rev):
-                # If rev is a tag or branch name try shallow clone first
-                from dvc.scm.base import CloneError
-
-                try:
-                    git = Git.clone(url, clone_path, shallow_branch=rev)
-                    shallow = True
-                    logger.debug(
-                        "erepo: using shallow clone for branch '%s'", rev
-                    )
-                except CloneError:
-                    pass
-            if not git:
-                git = Git.clone(url, clone_path)
-                shallow = False
-            CLONES[url] = (clone_path, shallow)
+            git = Git.clone(url, clone_path)
+            CLONES[url] = clone_path
     finally:
         if git:
             git.close()
 
-    return clone_path, shallow
-
-
-def _unshallow(git):
-    if git.gitpython.repo.head.is_detached:
-        # If this is a detached head (i.e. we shallow cloned a tag) switch to
-        # the default branch
-        origin_refs = git.gitpython.repo.remotes["origin"].refs
-        ref = origin_refs["HEAD"].reference
-        branch_name = ref.name.split("/")[-1]
-        branch = git.gitpython.repo.create_head(branch_name, ref)
-        branch.set_tracking_branch(ref)
-        branch.checkout()
-    git.pull(unshallow=True)
+    return clone_path
 
 
 def _git_checkout(repo_path, rev):
-    from dvc.scm.git import Git
-
     logger.debug("erepo: git checkout %s@%s", repo_path, rev)
     git = Git(repo_path)
     try:
@@ -247,16 +267,9 @@ def _git_checkout(repo_path, rev):
 
 
 def _remove(path):
-    from dvc.utils.fs import remove
-
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)
-        try:
-            os_retry(remove)(path)
-        except PermissionError:
-            logger.warning(
-                "Failed to remove '%s'", relpath(path), exc_info=True
-            )
+        os_retry(remove)(path)
     else:
         remove(path)

dvc/fs/__init__.py

@@ -1,90 +0,0 @@
-import posixpath
-from urllib.parse import urlparse
-
-from ..scheme import Schemes
-from .azure import AzureFileSystem
-from .gdrive import GDriveFileSystem
-from .gs import GSFileSystem
-from .hdfs import HDFSFileSystem
-from .http import HTTPFileSystem
-from .https import HTTPSFileSystem
-from .local import LocalFileSystem
-from .oss import OSSFileSystem
-from .s3 import S3FileSystem
-from .ssh import SSHFileSystem
-from .webdav import WebDAVFileSystem
-from .webdavs import WebDAVSFileSystem
-from .webhdfs import WebHDFSFileSystem
-
-FS_MAP = {
-    Schemes.AZURE: AzureFileSystem,
-    Schemes.GDRIVE: GDriveFileSystem,
-    Schemes.GS: GSFileSystem,
-    Schemes.HDFS: HDFSFileSystem,
-    Schemes.WEBHDFS: WebHDFSFileSystem,
-    Schemes.HTTP: HTTPFileSystem,
-    Schemes.HTTPS: HTTPSFileSystem,
-    Schemes.S3: S3FileSystem,
-    Schemes.SSH: SSHFileSystem,
-    Schemes.OSS: OSSFileSystem,
-    Schemes.WEBDAV: WebDAVFileSystem,
-    Schemes.WEBDAVS: WebDAVSFileSystem,
-    # NOTE: LocalFileSystem is the default
-}
-
-
-def get_fs_cls(remote_conf):
-    scheme = urlparse(remote_conf["url"]).scheme
-    return FS_MAP.get(scheme, LocalFileSystem)
-
-
-def get_fs_config(config, **kwargs):
-    name = kwargs.get("name")
-    if name:
-        remote_conf = config["remote"][name.lower()]
-    else:
-        remote_conf = kwargs
-    return _resolve_remote_refs(config, remote_conf)
-
-
-def _resolve_remote_refs(config, remote_conf):
-    # Support for cross referenced remotes.
-    # This will merge the settings, shadowing base ref with remote_conf.
-    # For example, having:
-    #
-    #       dvc remote add server ssh://localhost
-    #       dvc remote modify server user root
-    #       dvc remote modify server ask_password true
-    #
-    #       dvc remote add images remote://server/tmp/pictures
-    #       dvc remote modify images user alice
-    #       dvc remote modify images ask_password false
-    #       dvc remote modify images password asdf1234
-    #
-    # Results on a config dictionary like:
-    #
-    #       {
-    #           "url": "ssh://localhost/tmp/pictures",
-    #           "user": "alice",
-    #           "password": "asdf1234",
-    #           "ask_password": False,
-    #       }
-    parsed = urlparse(remote_conf["url"])
-    if parsed.scheme != "remote":
-        return remote_conf
-
-    base = get_fs_config(config, name=parsed.netloc)
-    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
-    return {**base, **remote_conf, "url": url}
-
-
-def get_cloud_fs(repo, **kwargs):
-    from dvc.config import ConfigError
-    from dvc.config_schema import SCHEMA, Invalid
-
-    remote_conf = get_fs_config(repo.config, **kwargs)
-    try:
-        remote_conf = SCHEMA["remote"][str](remote_conf)
-    except Invalid as exc:
-        raise ConfigError(str(exc)) from None
-    return get_fs_cls(remote_conf)(repo, remote_conf)

dvc/fs/_metadata.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, List
-
-from dvc.output import BaseOutput
-from dvc.path_info import PathInfo
-
-if TYPE_CHECKING:
-    from dvc.repo import Repo
-
-
-@dataclass
-class Metadata:
-    """
-    Container for storing metadata for a given path, similar in spirit to
-    `os.stat_result`.
-    """
-
-    # required field
-    path_info: PathInfo
-    repo: "Repo"
-
-    # computed fields
-    is_output: bool = field(init=False, default=False)  # is it an output?
-    # is the path part of an output?
-    part_of_output: bool = field(init=False, default=False)
-    # does the path contain outputs?
-    contains_outputs: bool = field(init=False, default=False)
-    # is the path tracked by dvc? equivalent to (is_output or part_of_output)
-    is_dvc: bool = field(init=False, default=False)
-    # does dvc has any output in that path or inside it? equiv. to bool(outs)
-    output_exists: bool = field(init=False)
-
-    # optional fields
-    isdir: bool = False  # is it a directory?
-    is_exec: bool = False  # is it an executable?
-    outs: List[BaseOutput] = field(default_factory=list)  # list of outputs
-
-    def __post_init__(self):
-        self.output_exists = bool(self.outs)
-
-        if not self.output_exists:
-            return
-
-        # it can contain multiple outputs
-        if len(self.outs) > 1:
-            self.contains_outputs = True
-        else:
-            out = self.outs[0]
-            # or, the path itself could be an output
-            self.is_output = self.path_info == out.path_info
-
-            # or, a directory must have an output somewhere deep inside it
-            if not self.is_output:
-                self.contains_outputs = out.path_info.isin(self.path_info)
-
-        # or, the path could be a part of an output, i.e. inside of an output
-        self.part_of_output = not self.is_output and not self.contains_outputs
-
-        self.is_dvc = self.is_output or self.part_of_output
-        # if it contains outputs, it must be a dir
-        self.isdir = self.contains_outputs
-
-    @property
-    def isfile(self):
-        # self.isdir might require additional computation, especially if it's
-        # a dir that's inside a dvc-tracked dir
-        # since, `Metadata()` does not support being used as null (i.e. when
-        # path does not exist), `isfile` becomes `isdir`'s complement.
-        return not self.isdir
-
-    def __str__(self):
-        return f"Metadata: {self.path_info}"

dvc/fs/azure.py

@@ -1,126 +0,0 @@
-import logging
-import os
-import threading
-
-from funcy import cached_property, wrap_prop
-
-from dvc.exceptions import DvcException
-from dvc.path_info import CloudURLInfo
-from dvc.scheme import Schemes
-from dvc.utils import format_link
-
-from .fsspec_wrapper import FSSpecWrapper
-
-logger = logging.getLogger(__name__)
-_DEFAULT_CREDS_STEPS = (
-    "https://azuresdkdocs.blob.core.windows.net/$web/python/"
-    "azure-identity/1.4.0/azure.identity.html#azure.identity"
-    ".DefaultAzureCredential"
-)
-
-
-class AzureAuthError(DvcException):
-    pass
-
-
-class AzureFileSystem(FSSpecWrapper):
-    scheme = Schemes.AZURE
-    PATH_CLS = CloudURLInfo
-    PARAM_CHECKSUM = "etag"
-    DETAIL_FIELDS = frozenset(("etag", "size"))
-    REQUIRES = {
-        "adlfs": "adlfs",
-        "knack": "knack",
-        "azure-identity": "azure.identity",
-    }
-
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
-
-        if not self.bucket:
-            container = self._az_config.get("storage", "container_name", None)
-            url = f"azure://{container}"
-
-        self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
-
-        self.login_method, self.login_info = self._prepare_credentials(config)
-
-    def _prepare_credentials(self, config):
-        from azure.identity.aio import DefaultAzureCredential
-
-        login_info = {}
-        login_info["connection_string"] = config.get(
-            "connection_string",
-            self._az_config.get("storage", "connection_string", None),
-        )
-        login_info["account_name"] = config.get(
-            "account_name", self._az_config.get("storage", "account", None)
-        )
-        login_info["account_key"] = config.get(
-            "account_key", self._az_config.get("storage", "key", None)
-        )
-        login_info["sas_token"] = config.get(
-            "sas_token", self._az_config.get("storage", "sas_token", None)
-        )
-        login_info["tenant_id"] = config.get("tenant_id")
-        login_info["client_id"] = config.get("client_id")
-        login_info["client_secret"] = config.get("client_secret")
-
-        if not any(login_info.values()):
-            login_info["credentials"] = DefaultAzureCredential()
-
-        for login_method, required_keys in [  # noqa
-            ("connection string", ["connection_string"]),
-            (
-                "AD service principal",
-                ["tenant_id", "client_id", "client_secret"],
-            ),
-            ("account key", ["account_name", "account_key"]),
-            ("SAS token", ["account_name", "sas_token"]),
-            ("anonymous login", ["account_name"]),
-            (f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
-        ]:
-            if all(login_info.get(key) is not None for key in required_keys):
-                break
-        else:
-            login_method = None
-
-        return login_method, login_info
-
-    @cached_property
-    def _az_config(self):
-        # NOTE: ideally we would've used get_default_cli().config from
-        # azure.cli.core, but azure-cli-core has a lot of conflicts with other
-        # dependencies. So instead we are just use knack directly
-        from knack.config import CLIConfig
-
-        config_dir = os.getenv(
-            "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
-        )
-        return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
-
-    @wrap_prop(threading.Lock())
-    @cached_property
-    def fs(self):
-        from adlfs import AzureBlobFileSystem
-        from azure.core.exceptions import AzureError
-
-        try:
-            file_system = AzureBlobFileSystem(**self.login_info)
-            if self.bucket not in [
-                container.rstrip("/") for container in file_system.ls("/")
-            ]:
-                file_system.mkdir(self.bucket)
-        except (ValueError, AzureError) as e:
-            raise AzureAuthError(
-                f"Authentication to Azure Blob Storage via {self.login_method}"
-                " failed.\nLearn more about configuration settings at"
-                f" {format_link('https://man.dvc.org/remote/modify')}"
-            ) from e
-
-        return file_system

dvc/fs/base.py

@@ -1,335 +0,0 @@
-import logging
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
-from multiprocessing import cpu_count
-from typing import Any, ClassVar, Dict, FrozenSet, Optional
-
-from funcy import cached_property
-
-from dvc.exceptions import DvcException
-from dvc.path_info import URLInfo
-from dvc.progress import Tqdm
-from dvc.utils import tmp_fname
-from dvc.utils.fs import makedirs, move
-from dvc.utils.http import open_url
-
-logger = logging.getLogger(__name__)
-
-
-class RemoteCmdError(DvcException):
-    def __init__(self, remote, cmd, ret, err):
-        super().__init__(
-            "{remote} command '{cmd}' finished with non-zero return code"
-            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
-        )
-
-
-class RemoteActionNotImplemented(DvcException):
-    def __init__(self, action, scheme):
-        m = f"{action} is not supported for {scheme} remotes"
-        super().__init__(m)
-
-
-class RemoteMissingDepsError(DvcException):
-    pass
-
-
-class BaseFileSystem:
-    scheme = "base"
-    REQUIRES: ClassVar[Dict[str, str]] = {}
-    PATH_CLS = URLInfo  # type: Any
-    JOBS = 4 * cpu_count()
-
-    CHECKSUM_DIR_SUFFIX = ".dir"
-    HASH_JOBS = max(1, min(4, cpu_count() // 2))
-    LIST_OBJECT_PAGE_SIZE = 1000
-    TRAVERSE_WEIGHT_MULTIPLIER = 5
-    TRAVERSE_PREFIX_LEN = 3
-    TRAVERSE_THRESHOLD_SIZE = 500000
-    CAN_TRAVERSE = True
-
-    # Needed for some providers, and http open()
-    CHUNK_SIZE = 64 * 1024 * 1024  # 64 MiB
-
-    PARAM_CHECKSUM: ClassVar[Optional[str]] = None
-    DETAIL_FIELDS: FrozenSet[str] = frozenset()
-
-    def __init__(self, repo, config):
-        self.repo = repo
-        self.config = config
-
-        self._check_requires()
-
-        self.path_info = None
-
-    @cached_property
-    def jobs(self):
-        return (
-            self.config.get("jobs")
-            or (self.repo and self.repo.config["core"].get("jobs"))
-            or self.JOBS
-        )
-
-    @cached_property
-    def hash_jobs(self):
-        return (
-            self.config.get("checksum_jobs")
-            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
-            or self.HASH_JOBS
-        )
-
-    @classmethod
-    def get_missing_deps(cls):
-        import importlib
-
-        missing = []
-        for package, module in cls.REQUIRES.items():
-            try:
-                importlib.import_module(module)
-            except ImportError:
-                missing.append(package)
-
-        return missing
-
-    def _check_requires(self):
-        from ..scheme import Schemes
-        from ..utils import format_link
-        from ..utils.pkg import PKG
-
-        missing = self.get_missing_deps()
-        if not missing:
-            return
-
-        url = self.config.get("url", f"{self.scheme}://")
-
-        scheme = self.scheme
-        if scheme == Schemes.WEBDAVS:
-            scheme = Schemes.WEBDAV
-
-        by_pkg = {
-            "pip": f"pip install 'dvc[{scheme}]'",
-            "conda": f"conda install -c conda-forge dvc-{scheme}",
-        }
-
-        cmd = by_pkg.get(PKG)
-        if cmd:
-            link = format_link("https://dvc.org/doc/install")
-            hint = (
-                f"To install dvc with those dependencies, run:\n"
-                "\n"
-                f"\t{cmd}\n"
-                "\n"
-                f"See {link} for more info."
-            )
-        else:
-            link = format_link("https://github.com/iterative/dvc/issues")
-            hint = f"Please report this bug to {link}. Thank you!"
-
-        raise RemoteMissingDepsError(
-            f"URL '{url}' is supported but requires these missing "
-            f"dependencies: {missing}. {hint}"
-        )
-
-    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
-        if hasattr(self, "_generate_download_url"):
-            # pylint:disable=no-member
-            func = self._generate_download_url  # type: ignore[attr-defined]
-            get_url = partial(func, path_info)
-            return open_url(get_url, mode=mode, encoding=encoding, **kwargs)
-
-        raise RemoteActionNotImplemented("open", self.scheme)
-
-    def exists(self, path_info, use_dvcignore=True) -> bool:
-        raise NotImplementedError
-
-    # pylint: disable=unused-argument
-
-    def isdir(self, path_info):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return False
-
-    def isfile(self, path_info):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between a directory and a file.
-        """
-        return True
-
-    def isexec(self, path_info):
-        """Optional: Overwrite only if the remote has a way to distinguish
-        between executable and non-executable file.
-        """
-        return False
-
-    def iscopy(self, path_info):
-        """Check if this file is an independent copy."""
-        return False  # We can't be sure by default
-
-    def walk_files(self, path_info, **kwargs):
-        """Return a generator with `PathInfo`s to all the files.
-
-        Optional kwargs:
-            prefix (bool): If true `path_info` will be treated as a prefix
-                rather than directory path.
-        """
-        raise NotImplementedError
-
-    def ls(self, path_info, detail=False, **kwargs):
-        raise RemoteActionNotImplemented("ls", self.scheme)
-
-    def is_empty(self, path_info):
-        return False
-
-    def info(self, path_info):
-        raise NotImplementedError
-
-    def getsize(self, path_info):
-        return self.info(path_info).get("size")
-
-    def remove(self, path_info):
-        raise RemoteActionNotImplemented("remove", self.scheme)
-
-    def makedirs(self, path_info):
-        """Optional: Implement only if the remote needs to create
-        directories before copying/linking/moving data
-        """
-
-    def move(self, from_info, to_info):
-        self.copy(from_info, to_info)
-        self.remove(from_info)
-
-    def copy(self, from_info, to_info):
-        raise RemoteActionNotImplemented("copy", self.scheme)
-
-    def symlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("symlink", self.scheme)
-
-    def hardlink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("hardlink", self.scheme)
-
-    def reflink(self, from_info, to_info):
-        raise RemoteActionNotImplemented("reflink", self.scheme)
-
-    # pylint: enable=unused-argument
-
-    @classmethod
-    def is_dir_hash(cls, hash_):
-        if not hash_:
-            return False
-        return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
-
-    def upload(
-        self, from_info, to_info, name=None, no_progress_bar=False,
-    ):
-        if not hasattr(self, "_upload"):
-            raise RemoteActionNotImplemented("upload", self.scheme)
-
-        if to_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        if from_info.scheme != "local":
-            raise NotImplementedError
-
-        logger.debug("Uploading '%s' to '%s'", from_info, to_info)
-
-        name = name or from_info.name
-
-        self._upload(  # noqa, pylint: disable=no-member
-            from_info.fspath,
-            to_info,
-            name=name,
-            no_progress_bar=no_progress_bar,
-        )
-
-    def upload_fobj(self, fobj, to_info, no_progress_bar=False, **pbar_args):
-        if not hasattr(self, "_upload_fobj"):
-            raise RemoteActionNotImplemented("upload_fobj", self.scheme)
-
-        with Tqdm.wrapattr(
-            fobj, "read", disable=no_progress_bar, bytes=True, **pbar_args
-        ) as wrapped:
-            self._upload_fobj(wrapped, to_info)  # pylint: disable=no-member
-
-    def download(
-        self,
-        from_info,
-        to_info,
-        name=None,
-        no_progress_bar=False,
-        jobs=None,
-        **kwargs,
-    ):
-        if not hasattr(self, "_download"):
-            raise RemoteActionNotImplemented("download", self.scheme)
-
-        if from_info.scheme != self.scheme:
-            raise NotImplementedError
-
-        if to_info.scheme == self.scheme != "local":
-            self.copy(from_info, to_info)
-            return 0
-
-        if to_info.scheme != "local":
-            raise NotImplementedError
-
-        if self.isdir(from_info):
-            return self._download_dir(
-                from_info, to_info, name, no_progress_bar, jobs, **kwargs,
-            )
-        return self._download_file(from_info, to_info, name, no_progress_bar,)
-
-    def _download_dir(
-        self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
-    ):
-        from_infos = list(self.walk_files(from_info, **kwargs))
-        to_infos = (
-            to_info / info.relative_to(from_info) for info in from_infos
-        )
-
-        with Tqdm(
-            total=len(from_infos),
-            desc="Downloading directory",
-            unit="Files",
-            disable=no_progress_bar,
-        ) as pbar:
-            download_files = pbar.wrap_fn(
-                partial(self._download_file, name=name, no_progress_bar=True,)
-            )
-            max_workers = jobs or self.jobs
-            with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                futures = [
-                    executor.submit(download_files, from_info, to_info)
-                    for from_info, to_info in zip(from_infos, to_infos)
-                ]
-
-                # NOTE: unlike pulling/fetching cache, where we need to
-                # download everything we can, not raising an error here might
-                # turn very ugly, as the user might think that he has
-                # downloaded a complete directory, while having a partial one,
-                # which might cause unexpected results in his pipeline.
-                for future in as_completed(futures):
-                    # NOTE: executor won't let us raise until all futures that
-                    # it has are finished, so we need to cancel them ourselves
-                    # before re-raising.
-                    exc = future.exception()
-                    if exc:
-                        for entry in futures:
-                            entry.cancel()
-                        raise exc
-
-    def _download_file(
-        self, from_info, to_info, name, no_progress_bar,
-    ):
-        makedirs(to_info.parent, exist_ok=True)
-
-        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
-        name = name or to_info.name
-
-        tmp_file = tmp_fname(to_info)
-
-        self._download(  # noqa, pylint: disable=no-member
-            from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
-        )
-
-        move(tmp_file, to_info)

dvc/fs/dvc.py

@@ -1,249 +0,0 @@
-import logging
-import os
-import typing
-
-from dvc.exceptions import OutputNotFoundError
-from dvc.path_info import PathInfo
-from dvc.utils import relpath
-
-from ._metadata import Metadata
-from .base import BaseFileSystem
-
-if typing.TYPE_CHECKING:
-    from dvc.output.base import BaseOutput
-
-
-logger = logging.getLogger(__name__)
-
-
-class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
-    """DVC repo fs.
-
-    Args:
-        repo: DVC repo.
-    """
-
-    scheme = "local"
-    PARAM_CHECKSUM = "md5"
-
-    def __init__(self, repo):
-        super().__init__(repo, {"url": repo.root_dir})
-
-    def _find_outs(self, path, *args, **kwargs):
-        outs = self.repo.find_outs_by_path(path, *args, **kwargs)
-
-        def _is_cached(out):
-            return out.use_cache
-
-        outs = list(filter(_is_cached, outs))
-        if not outs:
-            raise OutputNotFoundError(path, self.repo)
-
-        return outs
-
-    def _get_granular_hash(
-        self, path_info: PathInfo, out: "BaseOutput", remote=None
-    ):
-        assert isinstance(path_info, PathInfo)
-        # NOTE: use string paths here for performance reasons
-        key = tuple(relpath(path_info, out.path_info).split(os.sep))
-        out.get_dir_cache(remote=remote)
-        file_hash = out.hash_info.dir_info.get(key)
-        if file_hash:
-            return file_hash
-        raise FileNotFoundError
-
-    def open(  # type: ignore
-        self, path: PathInfo, mode="r", encoding=None, remote=None, **kwargs
-    ):  # pylint: disable=arguments-differ
-        try:
-            outs = self._find_outs(path, strict=False)
-        except OutputNotFoundError as exc:
-            raise FileNotFoundError from exc
-
-        if len(outs) != 1 or (
-            outs[0].is_dir_checksum and path == outs[0].path_info
-        ):
-            raise IsADirectoryError
-
-        out = outs[0]
-        if out.changed_cache(filter_info=path):
-            from dvc.config import NoRemoteError
-
-            try:
-                remote_obj = self.repo.cloud.get_remote(remote)
-            except NoRemoteError:
-                raise FileNotFoundError
-            if out.is_dir_checksum:
-                checksum = self._get_granular_hash(path, out).value
-            else:
-                checksum = out.hash_info.value
-            remote_info = remote_obj.odb.hash_to_path_info(checksum)
-            return remote_obj.fs.open(
-                remote_info, mode=mode, encoding=encoding
-            )
-
-        if out.is_dir_checksum:
-            checksum = self._get_granular_hash(path, out).value
-            cache_path = out.odb.hash_to_path_info(checksum).url
-        else:
-            cache_path = out.cache_path
-        return open(cache_path, mode=mode, encoding=encoding)
-
-    def exists(self, path):  # pylint: disable=arguments-differ
-        try:
-            self.metadata(path)
-            return True
-        except FileNotFoundError:
-            return False
-
-    def isdir(self, path):  # pylint: disable=arguments-differ
-        try:
-            meta = self.metadata(path)
-            return meta.isdir
-        except FileNotFoundError:
-            return False
-
-    def check_isdir(self, path_info, outs):
-        if len(outs) != 1:
-            return True
-
-        out = outs[0]
-        if not out.is_dir_checksum:
-            return out.path_info != path_info
-        if out.path_info == path_info:
-            return True
-
-        try:
-            self._get_granular_hash(path_info, out)
-            return False
-        except FileNotFoundError:
-            return True
-
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        try:
-            meta = self.metadata(path)
-            return meta.isfile
-        except FileNotFoundError:
-            return False
-
-    def _fetch_dir(self, out, **kwargs):
-        # pull dir cache if needed
-        out.get_dir_cache(**kwargs)
-
-        dir_cache = out.dir_cache
-        if not dir_cache:
-            raise FileNotFoundError
-
-        from dvc.objects import Tree
-
-        hash_info = Tree.save_dir_info(out.odb, dir_cache)
-        if hash_info != out.hash_info:
-            raise FileNotFoundError
-
-    def _add_dir(self, trie, out, **kwargs):
-        self._fetch_dir(out, **kwargs)
-
-        base = out.path_info.parts
-        for key, _ in out.dir_cache.items():  # noqa: B301
-            trie[base + key] = None
-
-    def _walk(self, root, trie, topdown=True, **kwargs):
-        dirs = set()
-        files = []
-
-        out = trie.get(root.parts)
-        if out and out.is_dir_checksum:
-            self._add_dir(trie, out, **kwargs)
-
-        root_len = len(root.parts)
-        for key, out in trie.iteritems(prefix=root.parts):  # noqa: B301
-            if key == root.parts:
-                continue
-
-            name = key[root_len]
-            if len(key) > root_len + 1 or (out and out.is_dir_checksum):
-                dirs.add(name)
-                continue
-
-            files.append(name)
-
-        assert topdown
-        dirs = list(dirs)
-        yield root.fspath, dirs, files
-
-        for dname in dirs:
-            yield from self._walk(root / dname, trie)
-
-    def walk(self, top, topdown=True, onerror=None, **kwargs):
-        from pygtrie import Trie
-
-        assert topdown
-        root = PathInfo(os.path.abspath(top))
-        try:
-            meta = self.metadata(root)
-        except FileNotFoundError:
-            if onerror is not None:
-                onerror(FileNotFoundError(top))
-            return
-
-        if not meta.isdir:
-            if onerror is not None:
-                onerror(NotADirectoryError(top))
-            return
-
-        trie = Trie()
-        for out in meta.outs:
-            trie[out.path_info.parts] = out
-
-            if out.is_dir_checksum and root.isin_or_eq(out.path_info):
-                self._add_dir(trie, out, **kwargs)
-
-        yield from self._walk(root, trie, topdown=topdown, **kwargs)
-
-    def walk_files(self, path_info, **kwargs):
-        for root, _, files in self.walk(path_info):
-            for fname in files:
-                # NOTE: os.path.join is ~5.5 times slower
-                yield PathInfo(f"{root}{os.sep}{fname}")
-
-    def isdvc(self, path, recursive=False, strict=True):
-        try:
-            meta = self.metadata(path)
-        except FileNotFoundError:
-            return False
-
-        recurse = recursive or not strict
-        return meta.output_exists if recurse else meta.is_output
-
-    def isexec(self, path_info):  # pylint: disable=unused-argument
-        return False
-
-    def metadata(self, path_info):
-        path_info = PathInfo(os.path.abspath(path_info))
-
-        try:
-            outs = self._find_outs(path_info, strict=False, recursive=True)
-        except OutputNotFoundError as exc:
-            raise FileNotFoundError from exc
-
-        meta = Metadata(path_info=path_info, outs=outs, repo=self.repo)
-        meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs)
-        return meta
-
-    def info(self, path_info):
-        meta = self.metadata(path_info)
-        ret = {"type": "dir" if meta.isdir else "file"}
-        if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
-            hash_info = meta.outs[0].hash_info
-            ret["size"] = hash_info.size
-            ret[hash_info.name] = hash_info.value
-        elif meta.part_of_output:
-            (out,) = meta.outs
-            key = path_info.relative_to(out.path_info).parts
-            hash_info = out.hash_info.dir_info.trie.get(key)
-            if hash_info:
-                ret["size"] = hash_info.size
-                ret[hash_info.name] = hash_info.value
-
-        return ret

dvc/fs/fsspec_wrapper.py

@@ -1,115 +0,0 @@
-import os
-import shutil
-
-from dvc.progress import Tqdm
-
-from .base import BaseFileSystem
-
-
-# pylint: disable=no-member
-class FSSpecWrapper(BaseFileSystem):
-    def fs(self):
-        raise NotImplementedError
-
-    def _with_bucket(self, path):
-        if isinstance(path, self.PATH_CLS):
-            return f"{path.bucket}/{path.path}"
-        return path
-
-    def _strip_bucket(self, entry):
-        _, entry = entry.split("/", 1)
-        return entry
-
-    def _strip_buckets(self, entries, detail, prefix=None):
-        for entry in entries:
-            if detail:
-                entry = self._entry_hook(entry.copy())
-                entry["name"] = self._strip_bucket(entry["name"])
-            else:
-                entry = self._strip_bucket(
-                    f"{prefix}/{entry}" if prefix else entry
-                )
-            yield entry
-
-    def _entry_hook(self, entry):
-        """Simple hook method to be overriden when wanted to process
-        entries within info() and ls(detail=True) calls"""
-        return entry
-
-    def isdir(self, path_info):
-        return self.fs.isdir(self._with_bucket(path_info))
-
-    def isfile(self, path_info):
-        return self.fs.isfile(self._with_bucket(path_info))
-
-    def open(
-        self, path_info, mode="r", **kwargs
-    ):  # pylint: disable=arguments-differ
-        return self.fs.open(self._with_bucket(path_info), mode=mode)
-
-    def copy(self, from_info, to_info):
-        self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
-
-    def exists(self, path_info, use_dvcignore=False):
-        return self.fs.exists(self._with_bucket(path_info))
-
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        path = self._with_bucket(path_info)
-        if recursive:
-            for root, _, files in self.fs.walk(path, detail=detail):
-                if detail:
-                    files = files.values()
-                yield from self._strip_buckets(files, detail, prefix=root)
-            return None
-
-        yield from self._strip_buckets(self.ls(path, detail=detail), detail)
-
-    def walk_files(self, path_info, **kwargs):
-        for file in self.ls(path_info, recursive=True):
-            yield path_info.replace(path=file)
-
-    def remove(self, path_info):
-        self.fs.rm(self._with_bucket(path_info))
-
-    def info(self, path_info):
-        info = self.fs.info(self._with_bucket(path_info)).copy()
-        info = self._entry_hook(info)
-        info["name"] = self._strip_bucket(info["name"])
-        return info
-
-    def _upload_fobj(self, fobj, to_info):
-        with self.open(to_info, "wb") as fdest:
-            shutil.copyfileobj(fobj, fdest, length=fdest.blocksize)
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **kwargs
-    ):
-        total = os.path.getsize(from_file)
-        with open(from_file, "rb") as fobj:
-            self.upload_fobj(
-                fobj,
-                self._with_bucket(to_info),
-                desc=name,
-                total=total,
-                no_progress_bar=no_progress_bar,
-            )
-        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
-
-    def _download(
-        self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
-    ):
-        total = self.getsize(self._with_bucket(from_info))
-        with self.open(from_info, "rb") as fobj:
-            with Tqdm.wrapattr(
-                fobj,
-                "read",
-                desc=name,
-                disable=no_progress_bar,
-                bytes=True,
-                total=total,
-                **pbar_args,
-            ) as wrapped:
-                with open(to_file, "wb") as fdest:
-                    shutil.copyfileobj(wrapped, fdest, length=fobj.blocksize)

dvc/fs/git.py

@@ -1,174 +0,0 @@
-import errno
-import os
-
-from funcy import cached_property
-
-from dvc.utils import is_exec, relpath
-
-from .base import BaseFileSystem
-
-
-class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
-    """Proxies the repo file access methods to Git objects"""
-
-    def __init__(
-        self, root_dir, trie, use_dvcignore=False, dvcignore_root=None
-    ):
-        super().__init__(None, {})
-        self._fs_root = root_dir
-        self.trie = trie
-        self.use_dvcignore = use_dvcignore
-        self.dvcignore_root = dvcignore_root
-
-    @property
-    def rev(self):
-        return self.trie.rev
-
-    @property
-    def fs_root(self):
-        return self._fs_root
-
-    def _get_key(self, path):
-        if isinstance(path, str):
-            if not os.path.isabs(path):
-                relparts = path.split(os.sep)
-            else:
-                relparts = relpath(path, self.fs_root).split(os.sep)
-        else:
-            relparts = path.relative_to(self.fs_root).parts
-        if relparts == ["."]:
-            return ()
-        return tuple(relparts)
-
-    @cached_property
-    def dvcignore(self):
-        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
-
-        root = self.dvcignore_root or self.fs_root
-        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
-        return cls(self, root)
-
-    def open(
-        self, path, mode="r", encoding=None
-    ):  # pylint: disable=arguments-differ
-        # NOTE: this is incorrect, we need to use locale to determine default
-        # encoding.
-        encoding = encoding or "utf-8"
-
-        key = self._get_key(path)
-        try:
-            return self.trie.open(key, mode=mode, encoding=encoding)
-        except KeyError as exc:
-            msg = os.strerror(errno.ENOENT) + f"in branch '{self.rev}'"
-            raise FileNotFoundError(errno.ENOENT, msg, path) from exc
-        except IsADirectoryError as exc:
-            raise IsADirectoryError(
-                errno.EISDIR, os.strerror(errno.EISDIR), path
-            ) from exc
-
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        def _is_ignored(path):
-            return self.dvcignore.is_ignored_file(
-                path
-            ) or self.dvcignore.is_ignored_dir(path)
-
-        if use_dvcignore and _is_ignored(path):
-            return False
-
-        key = self._get_key(path)
-        return self.trie.exists(key)
-
-    def isdir(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        if use_dvcignore and self.dvcignore.is_ignored_dir(path):
-            return False
-        key = self._get_key(path)
-        return self.trie.isdir(key)
-
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        if self.dvcignore.is_ignored_file(path):
-            return False
-        key = self._get_key(path)
-        return self.trie.isfile(key)
-
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        use_dvcignore=True,
-        ignore_subrepos=True,
-    ):
-        """Directory tree generator.
-
-        See `os.walk` for the docs. Differences:
-        - no support for symlinks
-        """
-        if not self.isdir(top, use_dvcignore=use_dvcignore):
-            if onerror:
-                if self.exists(top, use_dvcignore=use_dvcignore):
-                    exc = NotADirectoryError(
-                        errno.ENOTDIR, os.strerror(errno.ENOTDIR), top
-                    )
-                else:
-                    exc = FileNotFoundError(
-                        errno.ENOENT, os.strerror(errno.ENOENT), top
-                    )
-                onerror(exc)
-            return []
-
-        key = self._get_key(top)
-        for prefix, dirs, files in self.trie.walk(key, topdown=topdown):
-            if prefix:
-                root = os.path.join(self.fs_root, os.sep.join(prefix))
-            else:
-                root = self.fs_root
-            if use_dvcignore:
-                dirs[:], files[:] = self.dvcignore(
-                    root, dirs, files, ignore_subrepos=ignore_subrepos,
-                )
-            yield root, dirs, files
-
-    def isexec(self, path_info):
-        if not self.exists(path_info):
-            return False
-
-        mode = self.stat(path_info).st_mode
-        return is_exec(mode)
-
-    def info(self, path_info):
-        key = self._get_key(path_info)
-        try:
-            st = self.trie.stat(key)
-        except KeyError:
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), path_info
-            )
-        return {"size": st.st_size}
-
-    def stat(self, path):
-        key = self._get_key(path)
-        try:
-            return self.trie.stat(key)
-        except KeyError:
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), path
-            )
-
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        # NOTE: gitpython is not threadsafe. See
-        # https://github.com/iterative/dvc/issues/4079
-        return 1
-
-    def walk_files(self, top):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top):
-            for file in files:
-                # NOTE: os.path.join is ~5.5 times slower
-                yield f"{root}{os.sep}{file}"
-
-    def _reset(self):
-        return self.__dict__.pop("dvcignore", None)