dvc: bump to 1.3.0

pytest 6.0.0 had issue with `pylint`, complaining for `pytest.mark.*` being `not-callable`.

.codeclimate.yml

@@ -1,5 +0,0 @@
-languages:
-    Python: true
-
-exclude_paths:
-    - tests/

.deepsource.toml

@@ -1,15 +0,0 @@
-version = 1
-
-test_patterns = [
-  'tests/**'
-]
-
-exclude_patterns = [
-  'dvc/utils/compat.py'
-]
-
-[[analyzers]]
-name = 'python'
-enabled = true
-runtime_version = '3.x.x'
-

.dvc/.gitignore

@@ -9,3 +9,4 @@
 /pkg
 /repos
 /tmp
+/experiments

.flake8

@@ -1,7 +0,0 @@
-[flake8]
-ignore =
-    E203, # Whitespace before ':'
-    E266, # Too many leading '#' for block comment
-    W503, # Line break occurred before a binary operator
-max-line-length = 79
-select = B,C,E,F,W,T4,B9

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+patreon: DVCorg

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,10 +1,18 @@
 ---
-name: Bug report
-about: Create a report to help us improve
-
+name: "\U0001F41B Bug Report"
+about: Create a bug report to help us improve DVC
 ---
 
+## Bug Report
 
-**Please provide information about your setup**
-DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+### Please provide information about your setup
 
+**Output of `dvc version`:**
+
+```console
+$ dvc version
+```
+
+**Additional Information (if any):**
+
+If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: "🤗 Need help?"
+    url: https://dvc.org/chat
+    about: If you have a question, ask us on Discord. Please join with this invite 👉

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,5 @@
 ---
-name: Feature request
+name: "\U0001F680 Feature Request"
 about: Suggest an idea for this project
-
 ---
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,8 +1,5 @@
-* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
+* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
 
-* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
-
-* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
 
 Thank you for the contribution - we'll try to review it as soon as possible. 🙏
-

.github/workflows/check-patch.yaml

@@ -0,0 +1,19 @@
+name: Lint Python
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install requirements
+      run: bash ./scripts/ci/install.sh
+    - name: Check Patch
+      run: ./scripts/ci/check_patch.sh

.gitignore

@@ -26,7 +26,7 @@ innosetup/config.ini
 .coverage
 .coverage.*
 
-*.swp
+*.sw?
 
 pip-wheel-metadata/
 .vscode/

.pre-commit-config.yaml

@@ -1,11 +1,75 @@
 repos:
--   repo: https://github.com/ambv/black
-    rev: 19.3b0
+  - hooks:
+      - id: black
+        language_version: python3
+    repo: https://github.com/ambv/black
+    rev: 19.10b0
+  - repo: https://github.com/asottile/seed-isort-config
+    rev: v2.1.1
     hooks:
-    - id: black
-      language_version: python3
--   repo: https://gitlab.com/pycqa/flake8
+      - id: seed-isort-config
+  - hooks:
+      - id: isort
+        language_version: python3
+    repo: https://github.com/timothycrosley/isort
+    rev: 4.3.21
+  - hooks:
+      - id: flake8
+        language_version: python3
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-debugger
+          - flake8-string-format
+    repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.9
+  - repo: local
+    hooks:
+    - id: pylint
+      name: pylint
+      entry: pylint
+      language: system
+      types: [python]
+  - hooks:
+      - args:
+          - -i
+          - "2"
+        id: beautysh
+        language_version: python3
+    repo: https://github.com/lovesegfault/beautysh
     rev: master
-    hooks:
-    - id: flake8
-      language_version: python3
+  - hooks:
+      - args:
+          - git-hook
+          - pre-commit
+        entry: dvc
+        id: dvc-pre-commit
+        language: system
+        name: DVC pre-commit
+        stages:
+          - commit
+        verbose: true
+        require_serial: true
+      - args:
+          - git-hook
+          - pre-push
+        entry: dvc
+        id: dvc-pre-push
+        language: system
+        name: DVC pre-push
+        stages:
+          - push
+        require_serial: true
+      - always_run: true
+        args:
+          - git-hook
+          - post-checkout
+        entry: dvc
+        id: dvc-post-checkout
+        language: system
+        minimum_pre_commit_version: 2.2.0
+        name: DVC post-checkout
+        stages:
+          - post-checkout
+        require_serial: true
+    repo: local

.pre-commit-hooks.yaml

@@ -0,0 +1,36 @@
+- args:
+  - git-hook
+  - pre-commit
+  entry: dvc
+  id: dvc-pre-commit
+  language: python
+  language_version: python3
+  name: DVC pre-commit
+  require_serial: true
+  stages:
+  - commit
+  verbose: true
+- args:
+  - git-hook
+  - pre-push
+  entry: dvc
+  id: dvc-pre-push
+  language: python
+  language_version: python3
+  name: DVC pre-push
+  require_serial: true
+  stages:
+  - push
+- always_run: true
+  args:
+  - git-hook
+  - post-checkout
+  entry: dvc
+  id: dvc-post-checkout
+  language: python
+  language_version: python3
+  minimum_pre_commit_version: 2.2.0
+  name: DVC post-checkout
+  require_serial: true
+  stages:
+  - post-checkout

.pylintrc

@@ -15,7 +15,11 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-#init-hook=
+init-hook="import sys; \
+           from pathlib import Path; \
+           from pylint.config import find_pylintrc; \
+           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
@@ -28,7 +32,7 @@ limit-inference-results=100
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
-load-plugins=
+load-plugins=pylint_pytest,pylint_plugin_disable
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -60,7 +64,16 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use "--disable=all --enable=classes
 # --disable=W".
-disable=print-statement,
+disable=format,
+        # ^ most of them are covered by flake8,
+        # and are conventions rather than warnings or errors
+        refactoring,  # will be noisy if it starts giving suggestions
+        spelling,
+        # design is too-few-* and too-many-* checks, too-much-* for us right now
+        design,
+        # following were generated by default
+        # most of them are redundant Py3 checks for Py2+3 codebase
+        print-statement,
         parameter-unpacking,
         unpacking-in-except,
         old-raise-syntax,
@@ -139,9 +152,33 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        useless-object-inheritance,
-        missing-docstring,
-        bad-continuation
+        # custom disables start here
+        no-self-use,
+        invalid-name,
+        # already a check enabled for `wildcard-import`
+        unused-wildcard-import,
+        misplaced-comparison-constant,
+        # probably soon: https://github.com/iterative/dvc/issues/1843
+        logging-format-interpolation,
+        logging-fstring-interpolation,
+        # we don't have proper docstrings yet
+        # even if we do, we'll be doing it gradually
+        missing-function-docstring,
+        missing-module-docstring,
+        missing-class-docstring,
+        duplicate-code,
+        # too noisy, and detects issues even when import is inside a function
+        # see: https://github.com/PyCQA/pylint/issues/850
+        cyclic-import,
+        # isort should cover these:
+        wrong-import-order,
+        wrong-import-position,
+        ungrouped-imports,
+        multiple-imports,
+        # we use it to speedup/optimize
+        import-outside-toplevel,
+        fixme,
+
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -228,7 +265,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -249,7 +286,7 @@ ignore-on-opaque-inference=yes
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
+ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
@@ -290,7 +327,7 @@ dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore.
-ignored-argument-names=_.*|^ignored_|^unused_
+ignored-argument-names=_.*|^ignored_|^unused_|args|kwargs
 
 # Tells whether we should check for unused import in __init__ files.
 init-import=no

.restyled.yaml

@@ -1,7 +1,7 @@
 ---
 restylers:
   - name: black
-    image: restyled/restyler-black:v19.3b0
+    image: restyled/restyler-black:v19.10b0
     command:
     - black
     arguments: []
@@ -9,3 +9,12 @@ restylers:
     - "**/*.py"
     interpreters:
     - python
+  - name: isort
+    image: restyled/restyler-isort:v4.3.21
+    command:
+    - isort
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python

.travis.yml

@@ -29,7 +29,7 @@ jobs:
     os: linux
     language: python
     python: 3.7
-    before_install:
+    before_install: bash ./scripts/ci/install.sh
     install:
     script: ./scripts/ci/check_patch.sh
   # test jobs
@@ -37,15 +37,24 @@ jobs:
     stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
-  - name: "3.5 on Linux"
-    language: python
-    python: 3.5
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
+  - name: "3.8 on Windows"
+    os: windows           # Windows 10.0.17134 N/A Build 17134
+    language: shell       # 'language: python' is an error on Travis CI Windows
+    env:
+      - PYTHON_VERSION=3.8.0
+      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
   - name: "3.6 on Linux"
     language: python
     python: 3.6
   - name: "3.7 on Linux"
     language: python
     python: 3.7
+  - name: "3.8 on Linux"
+    language: python
+    python: 3.8
   - name: "3.7 on Mac"
     os: osx
     osx_image: xcode11.3
@@ -63,6 +72,9 @@ jobs:
     language: shell
     install:
     script: ./scripts/build_windows.cmd
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
   - name: "Linux pkgs"
     language: python
     python: 3.7
@@ -71,21 +83,27 @@ jobs:
     script: ./scripts/build_posix.sh
   - name: "PyPI pkgs"
     language: python
-    python: 3.7
+    python: 3.8
     script: ./scripts/build_package.sh
   - name: Snapcraft snap
     language: python
-    python: 3.7
+    python: 3.8
     addons:
      snaps:
      - name: snapcraft
        channel: stable
        confinement: classic
+     - name: review-tools
+       channel: stable
      - name: lxd
        channel: stable
     env:
-    - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
+    - SNAPCRAFT_IMAGE_INFO: |
+        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
+    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
+    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
     install:
+    - sudo usermod --append --groups lxd $USER
     - sudo /snap/bin/lxd.migrate -yes
     - sudo /snap/bin/lxd waitready
     - sudo /snap/bin/lxd init --auto
@@ -134,12 +152,11 @@ deploy:
       tags: true
       repo: iterative/dvc
       stage: build
-  - provider: snap
-    snap: dvc_*.snap
-    channel: $SNAP_CHANNEL
+  - provider: script
     skip_cleanup: true
+    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
     on:
       all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
+      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
       repo: iterative/dvc
       stage: build

.zenodo.json

@@ -0,0 +1,8 @@
+{
+  "title": "DVC: Data Version Control - Git for Data & Models",
+  "keywords": [
+    "data-science", "data-version-control", "machine-learning", "git",
+    "developer-tools", "reproducibility", "collaboration", "ai", "python"],
+  "contributors": [
+    {"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
+}

README.rst

@@ -1,6 +1,4 @@
-.. image:: https://dvc.org/static/img/logo-github-readme.png
-  :target: https://dvc.org
-  :alt: DVC logo
+|Banner|
 
 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
@@ -10,33 +8,9 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-.. image:: https://img.shields.io/badge/release-ok-brightgreen
-  :target: https://travis-ci.com/iterative/dvc
-  :alt: Release
+|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
 
-.. image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev
-  :target: https://travis-ci.com/iterative/dvc
-  :alt: Travis dev branch
-
-.. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
-  :target: https://codeclimate.com/github/iterative/dvc
-  :alt: Code Climate
-
-.. image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
-  :target: https://codecov.io/gh/iterative/dvc
-  :alt: Codecov
-
-.. image:: https://img.shields.io/badge/patreon-donate-green.svg
-  :target: https://www.patreon.com/DVCorg/overview
-  :alt: Donate
-
-.. image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
-  :target: https://anaconda.org/conda-forge/dvc
-  :alt: Conda-forge
-
-.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
-  :target: https://snapcraft.io/dvc
-  :alt: Snapcraft
+|PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
 
 |
 
@@ -79,9 +53,7 @@ to store data and model files seamlessly out of Git, while preserving almost the
 were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
 Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
-.. image:: https://dvc.org/static/img/flow.gif
-   :target: https://dvc.org/static/img/flow.gif
-   :alt: how_dvc_works
+|Flowchart|
 
 The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
 specify all steps required to produce a model: input dependencies including data, commands to run,
@@ -119,9 +91,51 @@ Installation
 There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
 Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.
 
+Snap (Snapcraft/Linux)
+----------------------
+
+|Snap|
+
+.. code-block:: bash
+
+   snap install dvc --classic
+
+This corresponds to the latest tagged release.
+Add ``--beta`` for the latest tagged release candidate,
+or ``--edge`` for the latest ``master`` version.
+
+Choco (Chocolatey/Windows)
+--------------------------
+
+|Choco|
+
+.. code-block:: bash
+
+   choco install dvc
+
+Brew (Homebrew/Mac OS)
+----------------------
+
+|Brew|
+
+.. code-block:: bash
+
+   brew install dvc
+
+Conda (Anaconda)
+----------------
+
+|Conda|
+
+.. code-block:: bash
+
+   conda install -c conda-forge dvc
+
 pip (PyPI)
 ----------
 
+|PyPI|
+
 .. code-block:: bash
 
    pip install dvc
@@ -137,36 +151,11 @@ To install the development version, run:
 
    pip install git+git://github.com/iterative/dvc
 
-Homebrew
---------
-
-.. code-block:: bash
-
-   brew install dvc
-
-
-Conda (Anaconda)
-----------------
-
-.. code-block:: bash
-
-   conda install -c conda-forge dvc
-
-Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
-
-Snap (Snapcraft)
-----------------
-
-.. code-block:: bash
-
-   snap install dvc --classic
-
-This corresponds to the latest tagged release.
-Add ``--edge`` for the latest ``master`` version.
-
 Package
 -------
 
+|Packages|
+
 Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
 can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
@@ -206,40 +195,43 @@ Comparison to related technologies
 
 Contributing
 ============
+
+|Maintainability| |Donate|
+
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
-  :alt: 0
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+   :alt: 0
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
-  :alt: 1
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+   :alt: 1
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
-  :alt: 2
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+   :alt: 2
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
-  :alt: 3
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+   :alt: 3
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
-  :alt: 4
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+   :alt: 4
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
-  :alt: 5
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+   :alt: 5
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
-  :alt: 6
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+   :alt: 6
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
-  :alt: 7
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+   :alt: 7
 
 Mailing List
 ============
@@ -253,3 +245,67 @@ This project is distributed under the Apache license version 2.0 (see the LICENS
 
 By submitting a pull request to this project, you agree to license your contribution under the Apache license version
 2.0 to this project.
+
+Citation
+========
+
+|DOI|
+
+Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
+`DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
+
+.. |Banner| image:: https://dvc.org/img/logo-github-readme.png
+   :target: https://dvc.org
+   :alt: DVC logo
+
+.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
+   :target: https://travis-ci.com/iterative/dvc/branches
+   :alt: Release
+
+.. |CI| image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev&logo=travis
+   :target: https://travis-ci.com/iterative/dvc/builds
+   :alt: Travis dev branch
+
+.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+   :target: https://codeclimate.com/github/iterative/dvc
+   :alt: Code Climate
+
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/iterative/dvc
+   :alt: Codecov
+
+.. |Donate| image:: https://img.shields.io/badge/patreon-donate-green.svg?logo=patreon
+   :target: https://www.patreon.com/DVCorg/overview
+   :alt: Donate
+
+.. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
+   :target: https://snapcraft.io/dvc
+   :alt: Snapcraft
+
+.. |Choco| image:: https://img.shields.io/chocolatey/v/dvc?label=choco
+   :target: https://chocolatey.org/packages/dvc
+   :alt: Chocolatey
+
+.. |Brew| image:: https://img.shields.io/homebrew/v/dvc?label=brew
+   :target: https://formulae.brew.sh/formula/dvc
+   :alt: Homebrew
+
+.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
+   :target: https://anaconda.org/conda-forge/dvc
+   :alt: Conda-forge
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/dvc.svg?label=pip&logo=PyPI&logoColor=white
+   :target: https://pypi.org/project/dvc
+   :alt: PyPI
+
+.. |Packages| image:: https://img.shields.io/github/v/release/iterative/dvc?label=deb|pkg|rpm|exe&logo=GitHub
+   :target: https://github.com/iterative/dvc/releases/latest
+   :alt: deb|pkg|rpm|exe
+
+.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.3677553-blue.svg
+   :target: https://doi.org/10.5281/zenodo.3677553
+   :alt: DOI
+
+.. |Flowchart| image:: https://dvc.org/img/flow.gif
+   :target: https://dvc.org/img/flow.gif
+   :alt: how_dvc_works

debian/changelog

@@ -2,4 +2,4 @@ dvc (0.75.b1) experimental; urgency=low
 
   * First native Debian package version.
 
- -- Dmitry Petrov <dmitry@dataversioncontrol.com>  Sat, 07 Dec 2019 09:49:28 +0300
+ -- Dmitry Petrov <dmitry@dvc.org>  Sat, 07 Dec 2019 09:49:28 +0300

debian/control

@@ -1,5 +1,5 @@
 Source: dvc
-Maintainer: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Maintainer: Dmitry Petrov <dmitry@dvc.org>
 Section: python
 Priority: optional
 Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)

debian/copyright

@@ -1,10 +1,10 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Upstream-Contact: Dmitry Petrov <dmitry@dvc.org>
 Source: https://github.com/iterative/dvc
 
 Files: *
-Copyright: Dmitry Petrov <dmitry@dataversioncontrol.com>
+Copyright: Dmitry Petrov <dmitry@dvc.org>
 License: Apache 2.0
  On Debian systems, the full text of the Apache 2.0
  License can be found in the file

dvc/__init__.py

@@ -3,17 +3,7 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-import warnings
-
 import dvc.logger
 from dvc.version import __version__  # noqa: F401
 
-
 dvc.logger.setup()
-
-# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
-# We don't directly import numpy, but our dependency networkx does, causing
-# these warnings in some environments. Luckily these warnings are benign and
-# we can simply ignore them so that they don't show up when you are using dvc.
-warnings.filterwarnings("ignore", message="numpy.dtype size changed")
-warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

dvc/analytics.py

@@ -2,12 +2,12 @@ import json
 import logging
 import os
 import platform
-import requests
 import sys
 import tempfile
 import uuid
 
 import distro
+import requests
 
 from dvc import __version__
 from dvc.config import Config, to_bool
@@ -15,11 +15,11 @@ from dvc.daemon import daemon
 from dvc.exceptions import NotDvcRepoError
 from dvc.lock import Lock, LockError
 from dvc.repo import Repo
-from dvc.scm import SCM
+from dvc.scm import SCM, NoSCM
+from dvc.scm.base import SCMError
 from dvc.utils import env2bool, is_binary
 from dvc.utils.fs import makedirs
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -35,7 +35,7 @@ def collect_and_send_report(args=None, return_code=None):
     report as a JSON, where the _collector_ generates it and the _sender_
     removes it after sending it.
     """
-    report = _runtime_info()
+    report = {}
 
     # Include command execution information on the report only when available.
     if args and hasattr(args, "func"):
@@ -54,17 +54,14 @@ def is_enabled():
         return False
 
     enabled = to_bool(
-        Config(validate=False)
-        .config.get(Config.SECTION_CORE, {})
-        .get(Config.SECTION_CORE_ANALYTICS, "true")
+        Config(validate=False).get("core", {}).get("analytics", "true")
     )
-
     logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
     return enabled
 
 
-def send(report):
+def send(path):
     """
     Side effect: Removes the report after sending it.
 
@@ -75,16 +72,25 @@ def send(report):
     url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
-    with open(report, "rb") as fobj:
-        requests.post(url, data=fobj, headers=headers, timeout=5)
+    with open(path) as fobj:
+        report = json.load(fobj)
 
-    os.remove(report)
+    report.update(_runtime_info())
+
+    try:
+        requests.post(url, json=report, headers=headers, timeout=5)
+    except requests.exceptions.RequestException:
+        logger.debug("failed to send analytics report", exc_info=True)
+
+    os.remove(path)
 
 
 def _scm_in_use():
     try:
         scm = SCM(root_dir=Repo.find_root())
         return type(scm).__name__
+    except SCMError:
+        return NoSCM.__name__
     except NotDvcRepoError:
         pass
 
@@ -141,7 +147,7 @@ def _find_or_create_user_id():
 
     IDs are generated randomly with UUID.
     """
-    config_dir = Config.get_global_config_dir()
+    config_dir = Config.get_dir("global")
     fname = os.path.join(config_dir, "user_id")
     lockfile = os.path.join(config_dir, "user_id.lock")
 
@@ -152,7 +158,7 @@ def _find_or_create_user_id():
     try:
         with Lock(lockfile):
             try:
-                with open(fname, "r") as fobj:
+                with open(fname) as fobj:
                     user_id = json.load(fobj)["user_id"]
 
             except (FileNotFoundError, ValueError, KeyError):
@@ -164,6 +170,4 @@ def _find_or_create_user_id():
             return user_id
 
     except LockError:
-        logger.debug(
-            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
-        )
+        logger.debug(f"Failed to acquire '{lockfile}'")

dvc/api.py

@@ -1,74 +1,53 @@
-from builtins import open as builtin_open
-import importlib
 import os
-import sys
-from contextlib import contextmanager, _GeneratorContextManager as GCM
-import threading
+from contextlib import _GeneratorContextManager as GCM
+from contextlib import contextmanager
 
-from funcy import wrap_with
-import ruamel.yaml
-from voluptuous import Schema, Required, Invalid
-
-from dvc.repo import Repo
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.external_repo import external_repo
-
-
-SUMMON_FILE_SCHEMA = Schema(
-    {
-        Required("objects"): [
-            {
-                Required("name"): str,
-                "meta": dict,
-                Required("summon"): {
-                    Required("type"): str,
-                    "deps": [str],
-                    str: object,
-                },
-            }
-        ]
-    }
-)
-SUMMON_PYTHON_SCHEMA = Schema(
-    {
-        Required("type"): "python",
-        Required("call"): str,
-        "args": dict,
-        "deps": [str],
-    }
-)
-
-
-class SummonError(DvcException):
-    pass
+from dvc.repo import Repo
 
 
 class UrlNotDvcRepoError(DvcException):
-    """Thrown if given url is not a DVC repository.
-
-    Args:
-        url (str): URL to the repository
-    """
+    """Thrown if the given URL is not a DVC repository."""
 
     def __init__(self, url):
-        super().__init__("'{}' is not a DVC repository.".format(url))
+        super().__init__(f"'{url}' is not a DVC repository.")
 
 
 def get_url(path, repo=None, rev=None, remote=None):
     """
-    Returns the full URL to the data artifact specified by its `path` in a
-    `repo`.
-    NOTE: There is no guarantee that the file actually exists in that location.
+    Returns the URL to the storage location of a data file or directory tracked
+    in a DVC repo. For Git repos, HEAD is used unless a rev argument is
+    supplied. The default remote is tried unless a remote argument is supplied.
+
+    Raises UrlNotDvcRepoError if repo is not a DVC project.
+
+    NOTE: This function does not check for the actual existence of the file or
+    directory in the remote storage.
     """
     with _make_repo(repo, rev=rev) as _repo:
-        _require_dvc(_repo)
+        if not isinstance(_repo, Repo):
+            raise UrlNotDvcRepoError(_repo.url)  # pylint: disable=no-member
         out = _repo.find_out_by_relpath(path)
         remote_obj = _repo.cloud.get_remote(remote)
-        return str(remote_obj.checksum_to_path_info(out.checksum))
+        return str(remote_obj.tree.hash_to_path_info(out.checksum))
 
 
-def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Context manager to open a file artifact as a file object."""
+def open(  # noqa, pylint: disable=redefined-builtin
+    path, repo=None, rev=None, remote=None, mode="r", encoding=None
+):
+    """
+    Open file in the supplied path tracked in a repo (both DVC projects and
+    plain Git repos are supported). For Git repos, HEAD is used unless a rev
+    argument is supplied. The default remote is tried unless a remote argument
+    is supplied. It may only be used as a context manager:
+
+        with dvc.api.open(
+                'path/to/file',
+                repo='https://example.com/url/to/repo'
+                ) as fd:
+            # ... Handle file object fd
+    """
     args = (path,)
     kwargs = {
         "repo": repo,
@@ -81,7 +60,9 @@ def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
 
 
 class _OpenContextManager(GCM):
-    def __init__(self, func, args, kwds):
+    def __init__(
+        self, func, args, kwds
+    ):  # pylint: disable=super-init-not-called
         self.gen = func(*args, **kwds)
         self.func, self.args, self.kwds = func, args, kwds
 
@@ -100,7 +81,11 @@ def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
 
 
 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Returns the contents of a file artifact."""
+    """
+    Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
+    is used unless a rev argument is supplied. The default remote is tried
+    unless a remote argument is supplied.
+    """
     with open(
         path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
     ) as fd:
@@ -118,130 +103,3 @@ def _make_repo(repo_url=None, rev=None):
             pass  # fallthrough to external_repo
     with external_repo(url=repo_url, rev=rev) as repo:
         yield repo
-
-
-def summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml", args=None):
-    """Instantiate an object described in the `summon_file`."""
-    with prepare_summon(
-        name, repo=repo, rev=rev, summon_file=summon_file
-    ) as desc:
-        try:
-            summon_dict = SUMMON_PYTHON_SCHEMA(desc.obj["summon"])
-        except Invalid as exc:
-            raise SummonError(str(exc)) from exc
-
-        _args = {**summon_dict.get("args", {}), **(args or {})}
-        return _invoke_method(summon_dict["call"], _args, desc.repo.root_dir)
-
-
-@contextmanager
-def prepare_summon(name, repo=None, rev=None, summon_file="dvcsummon.yaml"):
-    """Does a couple of things every summon needs as a prerequisite:
-    clones the repo, parses the summon file and pulls the deps.
-
-    Calling code is expected to complete the summon logic following
-    instructions stated in "summon" dict of the object spec.
-
-    Returns a SummonDesc instance, which contains references to a Repo object,
-    named object specification and resolved paths to deps.
-    """
-    with _make_repo(repo, rev=rev) as _repo:
-        _require_dvc(_repo)
-        try:
-            path = os.path.join(_repo.root_dir, summon_file)
-            obj = _get_object_spec(name, path)
-            yield SummonDesc(_repo, obj)
-        except SummonError as exc:
-            raise SummonError(
-                str(exc) + " at '{}' in '{}'".format(summon_file, repo)
-            ) from exc.__cause__
-
-
-class SummonDesc:
-    def __init__(self, repo, obj):
-        self.repo = repo
-        self.obj = obj
-        self._pull_deps()
-
-    @property
-    def deps(self):
-        return [os.path.join(self.repo.root_dir, d) for d in self._deps]
-
-    @property
-    def _deps(self):
-        return self.obj["summon"].get("deps", [])
-
-    def _pull_deps(self):
-        if not self._deps:
-            return
-
-        outs = [self.repo.find_out_by_relpath(d) for d in self._deps]
-
-        with self.repo.state:
-            for out in outs:
-                self.repo.cloud.pull(out.get_used_cache())
-                out.checkout()
-
-
-def _get_object_spec(name, path):
-    """
-    Given a summonable object's name, search for it on the given file
-    and return its description.
-    """
-    try:
-        with builtin_open(path, "r") as fobj:
-            content = SUMMON_FILE_SCHEMA(ruamel.yaml.safe_load(fobj.read()))
-            objects = [x for x in content["objects"] if x["name"] == name]
-
-        if not objects:
-            raise SummonError("No object with name '{}'".format(name))
-        elif len(objects) >= 2:
-            raise SummonError(
-                "More than one object with name '{}'".format(name)
-            )
-
-        return objects[0]
-
-    except FileNotFoundError as exc:
-        raise SummonError("Summon file not found") from exc
-    except ruamel.yaml.YAMLError as exc:
-        raise SummonError("Failed to parse summon file") from exc
-    except Invalid as exc:
-        raise SummonError(str(exc)) from exc
-
-
-@wrap_with(threading.Lock())
-def _invoke_method(call, args, path):
-    # XXX: Some issues with this approach:
-    #   * Import will pollute sys.modules
-    #   * sys.path manipulation is "theoretically" not needed,
-    #     but tests are failing for an unknown reason.
-    cwd = os.getcwd()
-
-    try:
-        os.chdir(path)
-        sys.path.insert(0, path)
-        method = _import_string(call)
-        return method(**args)
-    finally:
-        os.chdir(cwd)
-        sys.path.pop(0)
-
-
-def _import_string(import_name):
-    """Imports an object based on a string.
-    Useful to delay import to not load everything on startup.
-    Use dotted notaion in `import_name`, e.g. 'dvc.remote.gs.RemoteGS'.
-
-    :return: imported object
-    """
-    if "." in import_name:
-        module, obj = import_name.rsplit(".", 1)
-    else:
-        return importlib.import_module(import_name)
-    return getattr(importlib.import_module(module), obj)
-
-
-def _require_dvc(repo):
-    if not isinstance(repo, Repo):
-        raise UrlNotDvcRepoError(repo.url)

dvc/cache.py

@@ -1,136 +0,0 @@
-"""Manages cache of a DVC repo."""
-import os
-from collections import defaultdict
-
-from funcy import cached_property
-
-from dvc.config import Config
-
-
-class CacheConfig(object):
-    def __init__(self, config):
-        self.config = config
-
-    def set_dir(self, dname, level=None):
-        from dvc.remote.config import RemoteConfig
-
-        configobj = self.config.get_configobj(level)
-        path = RemoteConfig.resolve_path(dname, configobj.filename)
-        self.config.set(
-            Config.SECTION_CACHE, Config.SECTION_CACHE_DIR, path, level=level
-        )
-
-
-def _make_remote_property(name):
-    """
-    The config file is stored in a way that allows you to have a
-    cache for each remote.
-
-    This is needed when specifying external outputs
-    (as they require you to have an external cache location).
-
-    Imagine a config file like the following:
-
-            ['remote "dvc-storage"']
-            url = ssh://localhost/tmp
-            ask_password = true
-
-            [cache]
-            ssh = dvc-storage
-
-    This method creates a cached property, containing cache named `name`:
-
-        self.config == {'ssh': 'dvc-storage'}
-        self.ssh  # a RemoteSSH instance
-    """
-
-    def getter(self):
-        from dvc.remote import Remote
-
-        remote = self.config.get(name)
-        if not remote:
-            return None
-
-        return Remote(self.repo, name=remote)
-
-    getter.__name__ = name
-    return cached_property(getter)
-
-
-class Cache(object):
-    """Class that manages cache locations of a DVC repo.
-
-    Args:
-        repo (dvc.repo.Repo): repo instance that this cache belongs to.
-    """
-
-    CACHE_DIR = "cache"
-
-    def __init__(self, repo):
-        from dvc.remote import Remote
-
-        self.repo = repo
-
-        self.config = config = repo.config.config[Config.SECTION_CACHE]
-        local = config.get(Config.SECTION_CACHE_LOCAL)
-
-        if local:
-            name = Config.SECTION_REMOTE_FMT.format(local)
-            settings = repo.config.config[name]
-        else:
-            default_cache_dir = os.path.join(repo.dvc_dir, self.CACHE_DIR)
-            cache_dir = config.get(Config.SECTION_CACHE_DIR, default_cache_dir)
-            cache_type = config.get(Config.SECTION_CACHE_TYPE)
-            protected = config.get(Config.SECTION_CACHE_PROTECTED)
-            shared = config.get(Config.SECTION_CACHE_SHARED)
-
-            settings = {
-                Config.PRIVATE_CWD: config.get(
-                    Config.PRIVATE_CWD, repo.dvc_dir
-                ),
-                Config.SECTION_REMOTE_URL: cache_dir,
-                Config.SECTION_CACHE_TYPE: cache_type,
-                Config.SECTION_CACHE_PROTECTED: protected,
-                Config.SECTION_CACHE_SHARED: shared,
-            }
-
-        self.local = Remote(repo, **settings)
-
-    s3 = _make_remote_property(Config.SECTION_CACHE_S3)
-    gs = _make_remote_property(Config.SECTION_CACHE_GS)
-    ssh = _make_remote_property(Config.SECTION_CACHE_SSH)
-    hdfs = _make_remote_property(Config.SECTION_CACHE_HDFS)
-    azure = _make_remote_property(Config.SECTION_CACHE_AZURE)
-
-
-class NamedCache(object):
-    def __init__(self):
-        self._items = defaultdict(lambda: defaultdict(set))
-        self.external = defaultdict(set)
-
-    @classmethod
-    def make(cls, scheme, checksum, name):
-        cache = cls()
-        cache.add(scheme, checksum, name)
-        return cache
-
-    def __getitem__(self, key):
-        return self._items[key]
-
-    def add(self, scheme, checksum, name):
-        self._items[scheme][checksum].add(name)
-
-    def add_external(self, url, rev, path):
-        self.external[url, rev].add(path)
-
-    def update(self, cache, suffix=""):
-        for scheme, src in cache._items.items():
-            dst = self._items[scheme]
-            for checksum, names in src.items():
-                if suffix:
-                    dst[checksum].update(n + suffix for n in names)
-                else:
-                    dst[checksum].update(names)
-
-        for repo_pair, files in cache.external.items():
-            self.external[repo_pair].update(files)

dvc/cache/__init__.py

@@ -0,0 +1,171 @@
+"""Manages cache of a DVC repo."""
+from collections import defaultdict
+
+from funcy import cached_property
+
+
+def _make_remote_property(name):
+    """
+    The config file is stored in a way that allows you to have a
+    cache for each remote.
+
+    This is needed when specifying external outputs
+    (as they require you to have an external cache location).
+
+    Imagine a config file like the following:
+
+            ['remote "dvc-storage"']
+            url = ssh://localhost/tmp
+            ask_password = true
+
+            [cache]
+            ssh = dvc-storage
+
+    This method creates a cached property, containing cache named `name`:
+
+        self.config == {'ssh': 'dvc-storage'}
+        self.ssh  # a RemoteSSH instance
+    """
+
+    def getter(self):
+        from ..tree import get_cloud_tree
+        from .base import CloudCache
+
+        remote = self.config.get(name)
+        if not remote:
+            return None
+
+        tree = get_cloud_tree(self.repo, name=remote)
+        return CloudCache(tree)
+
+    getter.__name__ = name
+    return cached_property(getter)
+
+
+class Cache:
+    """Class that manages cache locations of a DVC repo.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that this cache belongs to.
+    """
+
+    CACHE_DIR = "cache"
+
+    def __init__(self, repo):
+        from ..tree import get_cloud_tree
+        from .local import LocalCache
+
+        self.repo = repo
+        self.config = config = repo.config["cache"]
+
+        local = config.get("local")
+
+        if local:
+            settings = {"name": local}
+        else:
+            settings = {**config, "url": config["dir"]}
+
+        tree = get_cloud_tree(repo, **settings)
+        self.local = LocalCache(tree)
+
+    s3 = _make_remote_property("s3")
+    gs = _make_remote_property("gs")
+    ssh = _make_remote_property("ssh")
+    hdfs = _make_remote_property("hdfs")
+    azure = _make_remote_property("azure")
+
+
+class NamedCacheItem:
+    def __init__(self):
+        self.names = set()
+        self.children = defaultdict(NamedCacheItem)
+
+    def __eq__(self, other):
+        return self.names == other.names and self.children == other.children
+
+    def child_keys(self):
+        for key, child in self.children.items():
+            yield key
+            yield from child.child_keys()
+
+    def child_names(self):
+        for key, child in self.children.items():
+            yield key, child.names
+            yield from child.child_names()
+
+    def add(self, checksum, item):
+        self.children[checksum].update(item)
+
+    def update(self, item, suffix=""):
+        if suffix:
+            self.names.update(n + suffix for n in item.names)
+        else:
+            self.names.update(item.names)
+        for checksum, child_item in item.children.items():
+            self.children[checksum].update(child_item)
+
+
+class NamedCache:
+    # pylint: disable=protected-access
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
+        self.external = defaultdict(set)
+
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
+
+    def __getitem__(self, key):
+        return self._items[key]
+
+    def add(self, scheme, checksum, name):
+        """Add a mapped name for the specified checksum."""
+        self._items[scheme][checksum].names.add(name)
+
+    def add_child_cache(self, checksum, cache, suffix=""):
+        """Add/update child cache for the specified checksum."""
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme][checksum].children
+            for child_checksum, item in src.items():
+                dst[child_checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)
+
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
+
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, item in src.items():
+                dst[checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)
+
+    def scheme_keys(self, scheme):
+        """Iterate over a flat list of all keys for the specified scheme,
+        including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key
+            yield from item.child_keys()
+
+    def scheme_names(self, scheme):
+        """Iterate over a flat list of checksum, names items for the specified
+        scheme, including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key, item.names
+            yield from item.child_names()
+
+    def dir_keys(self, scheme):
+        return (
+            key for key, item in self._items[scheme].items() if item.children
+        )
+
+    def child_keys(self, scheme, checksum):
+        return self._items[scheme][checksum].child_keys()

dvc/cache/base.py

@@ -0,0 +1,554 @@
+import json
+import logging
+from copy import copy
+
+from shortuuid import uuid
+
+import dvc.prompt as prompt
+from dvc.exceptions import CheckoutError, ConfirmRemoveError, DvcException
+from dvc.path_info import WindowsPathInfo
+from dvc.progress import Tqdm
+from dvc.remote.slow_link_detection import slow_link_guard
+
+from ..tree.base import RemoteActionNotImplemented
+
+logger = logging.getLogger(__name__)
+
+STATUS_OK = 1
+STATUS_MISSING = 2
+STATUS_NEW = 3
+STATUS_DELETED = 4
+
+STATUS_MAP = {
+    # (local_exists, remote_exists)
+    (True, True): STATUS_OK,
+    (False, False): STATUS_MISSING,
+    (True, False): STATUS_NEW,
+    (False, True): STATUS_DELETED,
+}
+
+
+class DirCacheError(DvcException):
+    def __init__(self, hash_):
+        super().__init__(
+            f"Failed to load dir cache for hash value: '{hash_}'."
+        )
+
+
+class CloudCache:
+    """Cloud cache class."""
+
+    DEFAULT_CACHE_TYPES = ["copy"]
+    CACHE_MODE = None
+
+    def __init__(self, tree):
+        self.tree = tree
+        self.repo = tree.repo
+
+        self.cache_types = tree.config.get("type") or copy(
+            self.DEFAULT_CACHE_TYPES
+        )
+        self.cache_type_confirmed = False
+        self._dir_info = {}
+
+    def get_dir_cache(self, hash_):
+        assert hash_
+
+        dir_info = self._dir_info.get(hash_)
+        if dir_info:
+            return dir_info
+
+        try:
+            dir_info = self.load_dir_cache(hash_)
+        except DirCacheError:
+            dir_info = []
+
+        self._dir_info[hash_] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, hash_):
+        path_info = self.tree.hash_to_path_info(hash_)
+
+        try:
+            with self.tree.open(path_info, "r") as fobj:
+                d = json.load(fobj)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(hash_) from exc
+
+        if not isinstance(d, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                path_info,
+            )
+            return []
+
+        if self.tree.PATH_CLS == WindowsPathInfo:
+            # only need to convert it for Windows
+            for info in d:
+                # NOTE: here is a BUG, see comment to .as_posix() below
+                relpath = info[self.tree.PARAM_RELPATH]
+                info[self.tree.PARAM_RELPATH] = relpath.replace(
+                    "/", self.tree.PATH_CLS.sep
+                )
+
+        return d
+
+    def changed(self, path_info, hash_info):
+        """Checks if data has changed.
+
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
+            - There's no file in the cache
+
+        Args:
+            path_info: dict with path information.
+            hash: expected hash value for this data.
+
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug(
+            "checking if '%s'('%s') has changed.", path_info, hash_info
+        )
+
+        if not self.tree.exists(path_info):
+            logger.debug("'%s' doesn't exist.", path_info)
+            return True
+
+        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
+        if hash_ is None:
+            logger.debug("hash value for '%s' is missing.", path_info)
+            return True
+
+        if self.changed_cache(hash_):
+            logger.debug("cache for '%s'('%s') has changed.", path_info, hash_)
+            return True
+
+        actual = self.tree.get_hash(path_info)
+        if hash_ != actual:
+            logger.debug(
+                "hash value '%s' for '%s' has changed (actual '%s').",
+                hash_,
+                actual,
+                path_info,
+            )
+            return True
+
+        logger.debug("'%s' hasn't changed.", path_info)
+        return False
+
+    def link(self, from_info, to_info):
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.tree.isfile(from_info)
+
+        self.tree.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self.tree, f"is_{link_type}", None)
+        if is_link and not is_link(path_info):
+            self.tree.remove(path_info)
+            raise DvcException(f"failed to verify {link_type}")
+
+        self.cache_type_confirmed = True
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self.tree, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self._verify_link(to_info, link_types[0])
+                return
+
+            except DvcException as exc:
+                logger.debug(
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
+                )
+                del link_types[0]
+
+        raise DvcException("no possible cache types left to try out.")
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.tree.exists(to_info):
+            raise DvcException(f"Link '{to_info}' already exists!")
+
+        link_method(from_info, to_info)
+
+        logger.debug(
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
+        )
+
+    def _save_file(self, path_info, tree, hash_, save_link=True, **kwargs):
+        assert hash_
+
+        cache_info = self.tree.hash_to_path_info(hash_)
+        if tree == self.tree:
+            if self.changed_cache(hash_):
+                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.link(cache_info, path_info)
+            elif self.tree.iscopy(path_info) and self._cache_is_copy(
+                path_info
+            ):
+                # Default relink procedure involves unneeded copy
+                self.tree.unprotect(path_info)
+            else:
+                self.tree.remove(path_info)
+                self.link(cache_info, path_info)
+
+            if save_link:
+                self.tree.state.save_link(path_info)
+            # we need to update path and cache, since in case of reflink,
+            # or copy cache type moving original file results in updates on
+            # next executed command, which causes md5 recalculation
+            self.tree.state.save(path_info, hash_)
+        else:
+            if self.changed_cache(hash_):
+                with tree.open(path_info, mode="rb") as fobj:
+                    # if tree has fetch enabled, DVC out will be fetched on
+                    # open and we do not need to read/copy any data
+                    if not (
+                        tree.isdvc(path_info, strict=False) and tree.fetch
+                    ):
+                        self.tree.copy_fobj(fobj, cache_info)
+                callback = kwargs.get("download_callback")
+                if callback:
+                    callback(1)
+
+        self.tree.state.save(cache_info, hash_)
+        return {self.tree.PARAM_CHECKSUM: hash_}
+
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.tree.path_info / ".cache_type_test_file"
+        if not self.tree.exists(test_cache_file):
+            with self.tree.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.tree.remove(workspace_file)
+            self.tree.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
+    def _save_dir(self, path_info, tree, hash_, save_link=True, **kwargs):
+        dir_info = self.get_dir_cache(hash_)
+        for entry in Tqdm(
+            dir_info, desc="Saving " + path_info.name, unit="file"
+        ):
+            entry_info = path_info / entry[self.tree.PARAM_RELPATH]
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+            self._save_file(
+                entry_info, tree, entry_hash, save_link=False, **kwargs
+            )
+
+        if save_link:
+            self.tree.state.save_link(path_info)
+        if self.tree.exists(path_info):
+            self.tree.state.save(path_info, hash_)
+
+        cache_info = self.tree.hash_to_path_info(hash_)
+        self.tree.state.save(cache_info, hash_)
+        return {self.tree.PARAM_CHECKSUM: hash_}
+
+    def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if path_info.scheme != self.tree.scheme:
+            raise RemoteActionNotImplemented(
+                f"save {path_info.scheme} -> {self.tree.scheme}",
+                self.tree.scheme,
+            )
+
+        if not hash_info:
+            hash_info = self.tree.save_info(path_info, tree=tree, **kwargs)
+        hash_ = hash_info[self.tree.PARAM_CHECKSUM]
+        return self._save(path_info, tree, hash_, save_link, **kwargs)
+
+    def _save(self, path_info, tree, hash_, save_link=True, **kwargs):
+        to_info = self.tree.hash_to_path_info(hash_)
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
+
+        if tree.isdir(path_info):
+            return self._save_dir(path_info, tree, hash_, save_link, **kwargs)
+        return self._save_file(path_info, tree, hash_, save_link, **kwargs)
+
+    # Override to return path as a string instead of PathInfo for clouds
+    # which support string paths (see local)
+    def hash_to_path(self, hash_):
+        return self.tree.hash_to_path_info(hash_)
+
+    def changed_cache_file(self, hash_):
+        """Compare the given hash with the (corresponding) actual one.
+
+        - Use `State` as a cache for computed hashes
+            + The entries are invalidated by taking into account the following:
+                * mtime
+                * inode
+                * size
+                * hash
+
+        - Remove the file from cache if it doesn't match the actual hash
+        """
+        # Prefer string path over PathInfo when possible due to performance
+        cache_info = self.hash_to_path(hash_)
+        if self.tree.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
+        actual = self.tree.get_hash(cache_info)
+
+        logger.debug(
+            "cache '%s' expected '%s' actual '%s'", cache_info, hash_, actual,
+        )
+
+        if not hash_ or not actual:
+            return True
+
+        if actual.split(".")[0] == hash_.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.tree.protect(cache_info)
+            return False
+
+        if self.tree.exists(cache_info):
+            logger.warning("corrupted cache file '%s'.", cache_info)
+            self.tree.remove(cache_info)
+
+        return True
+
+    def _changed_dir_cache(self, hash_, path_info=None, filter_info=None):
+        if self.changed_cache_file(hash_):
+            return True
+
+        for entry in self.get_dir_cache(hash_):
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+
+            if path_info and filter_info:
+                entry_info = path_info / entry[self.tree.PARAM_RELPATH]
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
+            if self.changed_cache_file(entry_hash):
+                return True
+
+        return False
+
+    def changed_cache(self, hash_, path_info=None, filter_info=None):
+        if self.tree.is_dir_hash(hash_):
+            return self._changed_dir_cache(
+                hash_, path_info=path_info, filter_info=filter_info
+            )
+        return self.changed_cache_file(hash_)
+
+    def already_cached(self, path_info):
+        current = self.tree.get_hash(path_info)
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.tree.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                " Are you sure you want to proceed?".format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise ConfirmRemoveError(str(path_info))
+
+        self.tree.remove(path_info)
+
+    def _checkout_file(
+        self, path_info, hash_, force, progress_callback=None, relink=False
+    ):
+        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
+        cache_info = self.tree.hash_to_path_info(hash_)
+        if self.tree.exists(path_info):
+            logger.debug("data '%s' will be replaced.", path_info)
+            self.safe_remove(path_info, force=force)
+            added, modified = False, True
+
+        self.link(cache_info, path_info)
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_)
+        if progress_callback:
+            progress_callback(str(path_info))
+
+        return added, modified and not relink
+
+    def _checkout_dir(
+        self,
+        path_info,
+        hash_,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        added, modified = False, False
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.tree.exists(path_info):
+            added = True
+            self.tree.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(hash_)
+
+        logger.debug("Linking directory '%s'.", path_info)
+
+        for entry in dir_info:
+            relative_path = entry[self.tree.PARAM_RELPATH]
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+            entry_cache_info = self.tree.hash_to_path_info(entry_hash)
+            entry_info = path_info / relative_path
+
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
+            entry_hash_info = {self.tree.PARAM_CHECKSUM: entry_hash}
+            if relink or self.changed(entry_info, entry_hash_info):
+                modified = True
+                self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.tree.state.save(entry_info, entry_hash)
+            if progress_callback:
+                progress_callback(str(entry_info))
+
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )
+
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_)
+
+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(self.tree.walk_files(path_info))
+
+        needed_files = {
+            path_info / entry[self.tree.PARAM_RELPATH] for entry in dir_info
+        }
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
+            self.safe_remove(path, force)
+
+        return bool(redundant_files)
+
+    def checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if path_info.scheme not in ["local", self.tree.scheme]:
+            raise NotImplementedError
+
+        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
+        failed = None
+        skip = False
+        if not hash_:
+            logger.warning(
+                "No file hash info found for '%s'. " "It won't be created.",
+                path_info,
+            )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        elif not relink and not self.changed(path_info, hash_info):
+            logger.debug("Data '%s' didn't change.", path_info)
+            skip = True
+
+        elif self.changed_cache(
+            hash_, path_info=path_info, filter_info=filter_info
+        ):
+            logger.warning(
+                "Cache '%s' not found. File '%s' won't be created.",
+                hash_,
+                path_info,
+            )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        if failed or skip:
+            if progress_callback:
+                progress_callback(
+                    str(path_info),
+                    self.get_files_number(
+                        self.tree.path_info, hash_, filter_info
+                    ),
+                )
+            if failed:
+                raise CheckoutError([failed])
+            return
+
+        logger.debug("Checking out '%s' with cache '%s'.", path_info, hash_)
+
+        return self._checkout(
+            path_info, hash_, force, progress_callback, relink, filter_info,
+        )
+
+    def _checkout(
+        self,
+        path_info,
+        hash_,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if not self.tree.is_dir_hash(hash_):
+            return self._checkout_file(
+                path_info, hash_, force, progress_callback, relink
+            )
+
+        return self._checkout_dir(
+            path_info, hash_, force, progress_callback, relink, filter_info
+        )
+
+    def get_files_number(self, path_info, hash_, filter_info):
+        from funcy.py3 import ilen
+
+        if not hash_:
+            return 0
+
+        if not self.tree.is_dir_hash(hash_):
+            return 1
+
+        if not filter_info:
+            return len(self.get_dir_cache(hash_))
+
+        return ilen(
+            filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
+            for entry in self.get_dir_cache(hash_)
+        )

dvc/cache/local.py

@@ -0,0 +1,431 @@
+import errno
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial, wraps
+
+from funcy import cached_property, concat
+
+from dvc.exceptions import DownloadError, UploadError
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+
+from ..remote.base import index_locked
+from ..tree.local import LocalTree
+from .base import (
+    STATUS_DELETED,
+    STATUS_MAP,
+    STATUS_MISSING,
+    STATUS_NEW,
+    CloudCache,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _log_exceptions(func, operation):
+    @wraps(func)
+    def wrapper(from_info, to_info, *args, **kwargs):
+        try:
+            func(from_info, to_info, *args, **kwargs)
+            return 0
+        except Exception as exc:  # pylint: disable=broad-except
+            # NOTE: this means we ran out of file descriptors and there is no
+            # reason to try to proceed, as we will hit this error anyways.
+            # pylint: disable=no-member
+            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+                raise
+
+            logger.exception(
+                "failed to %s '%s' to '%s'", operation, from_info, to_info
+            )
+            return 1
+
+    return wrapper
+
+
+class LocalCache(CloudCache):
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+    CACHE_MODE = LocalTree.CACHE_MODE
+
+    def __init__(self, tree):
+        super().__init__(tree)
+        self.cache_dir = tree.config.get("url")
+
+    @property
+    def cache_dir(self):
+        return self.tree.path_info.fspath if self.tree.path_info else None
+
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.tree.path_info = PathInfo(value) if value else None
+
+    @classmethod
+    def supported(cls, config):  # pylint: disable=unused-argument
+        return True
+
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def hash_to_path(self, hash_):
+        # NOTE: `self.cache_path` is already normalized so we can simply use
+        # `os.sep` instead of `os.path.join`. This results in this helper
+        # being ~5.5 times faster.
+        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
+
+    def hashes_exist(
+        self, hashes, jobs=None, name=None
+    ):  # pylint: disable=unused-argument
+        return [
+            hash_
+            for hash_ in Tqdm(
+                hashes,
+                unit="file",
+                desc="Querying "
+                + ("cache in " + name if name else "local cache"),
+            )
+            if not self.changed_cache_file(hash_)
+        ]
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        current_md5 = self.tree.get_hash(path_info)
+
+        if not current_md5:
+            return False
+
+        return not self.changed_cache(current_md5)
+
+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
+    @index_locked
+    def status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        # Return flattened dict containing all status info
+        dir_status, file_status, _ = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+        return dict(dir_status, **file_status)
+
+    def _status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
+
+        dir_status_info contains status for .dir files, file_status_info
+        contains status for all other files, and dir_contents is a dict of
+        {dir_hash: set(file_hash, ...)} which can be used to map
+        a .dir file to its file contents.
+        """
+        logger.debug(
+            f"Preparing to collect status from {remote.tree.path_info}"
+        )
+        md5s = set(named_cache.scheme_keys(self.tree.scheme))
+
+        logger.debug("Collecting information from local cache...")
+        local_exists = frozenset(
+            self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
+        )
+
+        # This is a performance optimization. We can safely assume that,
+        # if the resources that we want to fetch are already cached,
+        # there's no need to check the remote storage for the existence of
+        # those files.
+        if download and local_exists == md5s:
+            remote_exists = local_exists
+        else:
+            logger.debug("Collecting information from remote cache...")
+            remote_exists = set()
+            dir_md5s = set(named_cache.dir_keys(self.tree.scheme))
+            if dir_md5s:
+                remote_exists.update(
+                    self._indexed_dir_hashes(named_cache, remote, dir_md5s)
+                )
+                md5s.difference_update(remote_exists)
+            if md5s:
+                remote_exists.update(
+                    remote.hashes_exist(
+                        md5s, jobs=jobs, name=str(remote.tree.path_info)
+                    )
+                )
+        return self._make_status(
+            named_cache, show_checksums, local_exists, remote_exists
+        )
+
+    def _make_status(
+        self, named_cache, show_checksums, local_exists, remote_exists
+    ):
+        def make_names(hash_, names):
+            return {"name": hash_ if show_checksums else " ".join(names)}
+
+        dir_status = {}
+        file_status = {}
+        dir_contents = {}
+        for hash_, item in named_cache[self.tree.scheme].items():
+            if item.children:
+                dir_status[hash_] = make_names(hash_, item.names)
+                dir_contents[hash_] = set()
+                for child_hash, child in item.children.items():
+                    file_status[child_hash] = make_names(
+                        child_hash, child.names
+                    )
+                    dir_contents[hash_].add(child_hash)
+            else:
+                file_status[hash_] = make_names(hash_, item.names)
+
+        self._fill_statuses(dir_status, local_exists, remote_exists)
+        self._fill_statuses(file_status, local_exists, remote_exists)
+
+        self._log_missing_caches(dict(dir_status, **file_status))
+
+        return dir_status, file_status, dir_contents
+
+    def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
+        # Validate our index by verifying all indexed .dir hashes
+        # still exist on the remote
+        indexed_dirs = set(remote.index.dir_hashes())
+        indexed_dir_exists = set()
+        if indexed_dirs:
+            indexed_dir_exists.update(
+                remote.tree.list_hashes_exists(indexed_dirs)
+            )
+            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
+            if missing_dirs:
+                logger.debug(
+                    "Remote cache missing indexed .dir hashes '{}', "
+                    "clearing remote index".format(", ".join(missing_dirs))
+                )
+                remote.index.clear()
+
+        # Check if non-indexed (new) dir hashes exist on remote
+        dir_exists = dir_md5s.intersection(indexed_dir_exists)
+        dir_exists.update(
+            remote.tree.list_hashes_exists(dir_md5s - dir_exists)
+        )
+
+        # If .dir hash exists on the remote, assume directory contents
+        # still exists on the remote
+        for dir_hash in dir_exists:
+            file_hashes = list(
+                named_cache.child_keys(self.tree.scheme, dir_hash)
+            )
+            if dir_hash not in remote.index:
+                logger.debug(
+                    "Indexing new .dir '{}' with '{}' nested files".format(
+                        dir_hash, len(file_hashes)
+                    )
+                )
+                remote.index.update([dir_hash], file_hashes)
+            yield dir_hash
+            yield from file_hashes
+
+    @staticmethod
+    def _fill_statuses(hash_info_dir, local_exists, remote_exists):
+        # Using sets because they are way faster for lookups
+        local = set(local_exists)
+        remote = set(remote_exists)
+
+        for md5, info in hash_info_dir.items():
+            status = STATUS_MAP[(md5 in local, md5 in remote)]
+            info["status"] = status
+
+    def _get_plans(self, download, remote, status_info, status):
+        cache = []
+        path_infos = []
+        names = []
+        hashes = []
+        for md5, info in Tqdm(
+            status_info.items(), desc="Analysing status", unit="file"
+        ):
+            if info["status"] == status:
+                cache.append(self.tree.hash_to_path_info(md5))
+                path_infos.append(remote.tree.hash_to_path_info(md5))
+                names.append(info["name"])
+                hashes.append(md5)
+
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return from_infos, to_infos, names, hashes
+
+    def _process(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        logger.debug(
+            "Preparing to {} '{}'".format(
+                "download data from" if download else "upload data to",
+                remote.tree.path_info,
+            )
+        )
+
+        if download:
+            func = partial(
+                _log_exceptions(remote.tree.download, "download"),
+                dir_mode=self.tree.dir_mode,
+                file_mode=self.tree.file_mode,
+            )
+            status = STATUS_DELETED
+            desc = "Downloading"
+        else:
+            func = _log_exceptions(remote.tree.upload, "upload")
+            status = STATUS_NEW
+            desc = "Uploading"
+
+        if jobs is None:
+            jobs = remote.tree.JOBS
+
+        dir_status, file_status, dir_contents = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+
+        dir_plans = self._get_plans(download, remote, dir_status, status)
+        file_plans = self._get_plans(download, remote, file_status, status)
+
+        total = len(dir_plans[0]) + len(file_plans[0])
+        if total == 0:
+            return 0
+
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
+                if download:
+                    from_infos, to_infos, names, _ = (
+                        d + f for d, f in zip(dir_plans, file_plans)
+                    )
+                    fails = sum(
+                        executor.map(func, from_infos, to_infos, names)
+                    )
+                else:
+                    # for uploads, push files first, and any .dir files last
+
+                    file_futures = {}
+                    for from_info, to_info, name, hash_ in zip(*file_plans):
+                        file_futures[hash_] = executor.submit(
+                            func, from_info, to_info, name
+                        )
+                    dir_futures = {}
+                    for from_info, to_info, name, dir_hash in zip(*dir_plans):
+                        wait_futures = {
+                            future
+                            for file_hash, future in file_futures.items()
+                            if file_hash in dir_contents[dir_hash]
+                        }
+                        dir_futures[dir_hash] = executor.submit(
+                            self._dir_upload,
+                            func,
+                            wait_futures,
+                            from_info,
+                            to_info,
+                            name,
+                        )
+                    fails = sum(
+                        future.result()
+                        for future in concat(
+                            file_futures.values(), dir_futures.values()
+                        )
+                    )
+
+        if fails:
+            if download:
+                remote.index.clear()
+                raise DownloadError(fails)
+            raise UploadError(fails)
+
+        if not download:
+            # index successfully pushed dirs
+            for dir_hash, future in dir_futures.items():
+                if future.result() == 0:
+                    file_hashes = dir_contents[dir_hash]
+                    logger.debug(
+                        "Indexing pushed dir '{}' with "
+                        "'{}' nested files".format(dir_hash, len(file_hashes))
+                    )
+                    remote.index.update([dir_hash], file_hashes)
+
+        return len(dir_plans[0]) + len(file_plans[0])
+
+    @staticmethod
+    def _dir_upload(func, futures, from_info, to_info, name):
+        for future in as_completed(futures):
+            if future.result():
+                # do not upload this .dir file if any file in this
+                # directory failed to upload
+                logger.debug(
+                    "failed to upload full contents of '{}', "
+                    "aborting .dir file upload".format(name)
+                )
+                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
+                return 1
+        return func(from_info, to_info, name)
+
+    @index_locked
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
+        return self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
+
+    @index_locked
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
+        return self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )
+
+    @staticmethod
+    def _log_missing_caches(hash_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in hash_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "\n".join(
+                "name: {}, md5: {}".format(info["name"], md5)
+                for md5, info in missing_caches
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files:\n{}".format(missing_desc)
+            )
+            logger.warning(msg)

dvc/cli.py

@@ -3,38 +3,44 @@ import argparse
 import logging
 import sys
 
-import dvc.command.add as add
-import dvc.command.cache as cache
-import dvc.command.checkout as checkout
-import dvc.command.commit as commit
-import dvc.command.config as config
-import dvc.command.daemon as daemon
-import dvc.command.data_sync as data_sync
-import dvc.command.destroy as destroy
-import dvc.command.diff as diff
-import dvc.command.gc as gc
-import dvc.command.get as get
-import dvc.command.get_url as get_url
-import dvc.command.imp as imp
-import dvc.command.imp_url as imp_url
-import dvc.command.init as init
-import dvc.command.install as install
-import dvc.command.lock as lock
-import dvc.command.metrics as metrics
-import dvc.command.move as move
-import dvc.command.pipeline as pipeline
-import dvc.command.remote as remote
-import dvc.command.remove as remove
-import dvc.command.repro as repro
-import dvc.command.root as root
-import dvc.command.run as run
-import dvc.command.tag as tag
-import dvc.command.unprotect as unprotect
-import dvc.command.update as update
-import dvc.command.version as version
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcParserError
-
+from .command import (
+    add,
+    cache,
+    checkout,
+    commit,
+    completion,
+    config,
+    daemon,
+    dag,
+    data_sync,
+    destroy,
+    diff,
+    experiments,
+    freeze,
+    gc,
+    get,
+    get_url,
+    git_hook,
+    imp,
+    imp_url,
+    init,
+    install,
+    ls,
+    metrics,
+    move,
+    params,
+    plots,
+    remote,
+    remove,
+    repro,
+    root,
+    run,
+    unprotect,
+    update,
+    version,
+)
+from .command.base import fix_subparsers
+from .exceptions import DvcParserError
 
 logger = logging.getLogger(__name__)
 
@@ -58,38 +64,60 @@ COMMANDS = [
     remote,
     cache,
     metrics,
+    params,
     install,
     root,
-    lock,
-    pipeline,
+    ls,
+    freeze,
+    dag,
     daemon,
     commit,
-    tag,
+    completion,
     diff,
     version,
     update,
+    git_hook,
+    plots,
+    experiments,
 ]
 
 
+def _find_parser(parser, cmd_cls):
+    defaults = parser._defaults  # pylint: disable=protected-access
+    if not cmd_cls or cmd_cls == defaults.get("func"):
+        parser.print_help()
+        raise DvcParserError()
+
+    actions = parser._actions  # pylint: disable=protected-access
+    for action in actions:
+        if not isinstance(action.choices, dict):
+            # NOTE: we are only interested in subparsers
+            continue
+        for subparser in action.choices.values():
+            _find_parser(subparser, cmd_cls)
+
+
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
 
-    def error(self, message):
-        """Custom error method.
-        Args:
-            message (str): error message.
-
-        Raises:
-            dvc.exceptions.DvcParser: dvc parser exception.
-        """
+    def error(self, message, cmd_cls=None):  # pylint: disable=arguments-differ
         logger.error(message)
-        self.print_help()
-        raise DvcParserError()
+        _find_parser(self, cmd_cls)
+
+    def parse_args(self, args=None, namespace=None):
+        # NOTE: overriding to provide a more granular help message.
+        # E.g. `dvc plots diff --bad-flag` would result in a `dvc plots diff`
+        # help message instead of generic `dvc` usage.
+        args, argv = self.parse_known_args(args, namespace)
+        if argv:
+            msg = "unrecognized arguments: %s"
+            self.error(msg % " ".join(argv), getattr(args, "func", None))
+        return args
 
 
 class VersionAction(argparse.Action):  # pragma: no cover
     # pylint: disable=too-few-public-methods
-    """Shows dvc version and exits."""
+    """Shows DVC version and exits."""
 
     def __call__(self, parser, namespace, values, option_string=None):
         from dvc import __version__
@@ -109,28 +137,16 @@ def get_parent_parser():
 
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
-        "-q", "--quiet", action="store_true", default=False, help="Be quiet."
+        "-q", "--quiet", action="count", default=0, help="Be quiet."
     )
     log_level_group.add_argument(
-        "-v",
-        "--verbose",
-        action="store_true",
-        default=False,
-        help="Be verbose.",
+        "-v", "--verbose", action="count", default=0, help="Be verbose."
     )
 
     return parent_parser
 
 
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        dvc.exceptions.DvcParserError: raised for argument parsing errors.
-    """
+def get_main_parser():
     parent_parser = get_parent_parser()
 
     # Main parser
@@ -178,6 +194,18 @@ def parse_args(argv=None):
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
-    args = parser.parse_args(argv)
+    return parser
 
+
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
     return args

dvc/command/add.py

@@ -1,11 +1,9 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-from dvc.exceptions import RecursiveAddingWhileUsingFilename
-
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +19,7 @@ class CmdAdd(CmdBase):
                 recursive=self.args.recursive,
                 no_commit=self.args.no_commit,
                 fname=self.args.file,
+                external=self.args.external,
             )
 
         except DvcException:
@@ -53,11 +52,17 @@ def add_parser(subparsers, parent_parser):
         help="Don't put files/directories into cache.",
     )
     parser.add_argument(
-        "-f",
-        "--file",
-        help="Specify name of the DVC-file this command will generate.",
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow targets that are outside of the DVC repository.",
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add."
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
     )
+    parser.add_argument(
+        "targets", nargs="+", help="Input files/directories to add.",
+    ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,5 +1,5 @@
 import logging
-
+from abc import ABC, abstractmethod
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +27,7 @@ def append_doc_link(help_message, path):
     )
 
 
-class CmdBase(object):
+class CmdBase(ABC):
     def __init__(self, args):
         from dvc.repo import Repo
         from dvc.updater import Updater
@@ -35,24 +35,23 @@ class CmdBase(object):
         self.repo = Repo()
         self.config = self.repo.config
         self.args = args
-        hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
-        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
+        hardlink_lock = self.config["core"].get("hardlink_lock", False)
+        updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
         updater.check()
 
     @property
     def default_targets(self):
-        """Default targets for `dvc repro` and `dvc pipeline`."""
-        from dvc.stage import Stage
+        """Default targets for `dvc repro`."""
+        from dvc.dvcfile import PIPELINE_FILE
 
-        msg = "assuming default target '{}'.".format(Stage.STAGE_FILE)
-        logger.warning(msg)
-        return [Stage.STAGE_FILE]
+        logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
+        return [PIPELINE_FILE]
 
-    # Abstract methods that have to be implemented by any inheritance class
+    @abstractmethod
     def run(self):
-        raise NotImplementedError
+        pass
 
 
 class CmdBaseNoRepo(CmdBase):
-    def __init__(self, args):
+    def __init__(self, args):  # pylint: disable=super-init-not-called
         self.args = args

dvc/command/cache.py

@@ -1,18 +1,14 @@
 import argparse
 
-from dvc.cache import CacheConfig
-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.command import completion
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
 
 class CmdCacheDir(CmdConfig):
-    def __init__(self, args):
-        super().__init__(args)
-        self.cache_config = CacheConfig(self.config)
-
     def run(self):
-        self.cache_config.set_dir(self.args.value, level=self.args.level)
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
         return 0
 
 
@@ -60,5 +56,5 @@ def add_parser(subparsers, parent_parser):
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
         "config file location.",
-    )
+    ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -1,18 +1,68 @@
 import argparse
+import logging
+import operator
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+import colorama
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import CheckoutError
+from dvc.utils.humanize import get_summary
+
+logger = logging.getLogger(__name__)
+
+
+def log_changes(stats):
+    colors = [
+        ("modified", colorama.Fore.YELLOW,),
+        ("added", colorama.Fore.GREEN),
+        ("deleted", colorama.Fore.RED,),
+    ]
+
+    for state, color in colors:
+        entries = stats.get(state)
+
+        if not entries:
+            continue
+
+        for entry in entries:
+            logger.info(
+                "{color}{state}{nc}{spacing}{entry}".format(
+                    color=color,
+                    state=state[0].upper(),
+                    nc=colorama.Fore.RESET,
+                    spacing="\t",
+                    entry=entry,
+                )
+            )
 
 
 class CmdCheckout(CmdBase):
     def run(self):
-        self.repo.checkout(
-            targets=self.args.targets,
-            with_deps=self.args.with_deps,
-            force=self.args.force,
-            relink=self.args.relink,
-            recursive=self.args.recursive,
-        )
+        stats, exc = None, None
+        try:
+            stats = self.repo.checkout(
+                targets=self.args.targets,
+                with_deps=self.args.with_deps,
+                force=self.args.force,
+                relink=self.args.relink,
+                recursive=self.args.recursive,
+            )
+        except CheckoutError as _exc:
+            exc = _exc
+            stats = exc.stats
+
+        if self.args.summary:
+            default_message = "No changes."
+            msg = get_summary(
+                sorted(stats.items(), key=operator.itemgetter(0))
+            )
+            logger.info(msg or default_message)
+        else:
+            log_changes(stats)
+
+        if exc:
+            raise exc
         return 0
 
 
@@ -26,6 +76,12 @@ def add_parser(subparsers, parent_parser):
         help=CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    checkout_parser.add_argument(
+        "--summary",
+        action="store_true",
+        default=False,
+        help="Show summary of the changes.",
+    )
     checkout_parser.add_argument(
         "-d",
         "--with-deps",
@@ -56,7 +112,9 @@ def add_parser(subparsers, parent_parser):
     checkout_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-files to checkout. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -47,7 +46,7 @@ def add_parser(subparsers, parent_parser):
         "--force",
         action="store_true",
         default=False,
-        help="Commit even if checksums for dependencies/outputs changed.",
+        help="Commit even if hash value for dependencies/outputs changed.",
     )
     commit_parser.add_argument(
         "-d",
@@ -68,5 +67,5 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to commit. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
-    )
+    ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -0,0 +1,57 @@
+import argparse
+import logging
+
+import shtab
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+
+logger = logging.getLogger(__name__)
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
+}
+PREAMBLE = {
+    "bash": """
+# $1=COMP_WORDS[1]
+_dvc_compgen_DVCFiles() {
+  compgen -d -S '/' -- $1  # recurse into subdirs
+  compgen -f -X '!*?.dvc' -- $1
+  compgen -f -X '!*Dvcfile' -- $1
+  compgen -f -X '!*dvc.yaml' -- $1
+}
+""",
+    "zsh": "",
+}
+
+
+class CmdCompletion(CmdBaseNoRepo):
+    def run(self):
+        from dvc.cli import get_main_parser
+
+        parser = get_main_parser()
+        shell = self.args.shell
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
+        print(script)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMPLETION_HELP = "Generate shell tab completion."
+    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
+    completion_parser = subparsers.add_parser(
+        "completion",
+        parents=[parent_parser],
+        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
+        help=COMPLETION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    completion_parser.add_argument(
+        "-s",
+        "--shell",
+        help="Shell syntax for completions.",
+        default="bash",
+        choices=["bash", "zsh"],
+    )
+    completion_parser.set_defaults(func=CmdCompletion)

dvc/command/config.py

@@ -1,10 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-from dvc.config import Config
-
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.config import Config, ConfigError
 
 logger = logging.getLogger(__name__)
 
@@ -18,17 +16,21 @@ class CmdConfig(CmdBaseNoRepo):
     def run(self):
         section, opt = self.args.name.lower().strip().split(".", 1)
 
-        if self.args.unset:
-            self.config.unset(section, opt, level=self.args.level)
-        elif self.args.value is None:
-            logger.info(self.config.get(section, opt, level=self.args.level))
-        else:
-            self.config.set(
-                section, opt, self.args.value, level=self.args.level
-            )
+        if self.args.value is None and not self.args.unset:
+            conf = self.config.load_one(self.args.level)
+            self._check(conf, section, opt)
+            logger.info(conf[section][opt])
+            return 0
 
-        is_write = self.args.unset or self.args.value is not None
-        if is_write and self.args.name == "cache.type":
+        with self.config.edit(self.args.level) as conf:
+            if self.args.unset:
+                self._check(conf, section, opt)
+                del conf[section][opt]
+            else:
+                self._check(conf, section)
+                conf[section][opt] = self.args.value
+
+        if self.args.name == "cache.type":
             logger.warning(
                 "You have changed the 'cache.type' option. This doesn't update"
                 " any existing workspace file links, but it can be done with:"
@@ -37,30 +39,40 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
+    def _check(self, conf, section, opt=None):
+        if section not in conf:
+            msg = "section {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
+        if opt and opt not in conf[section]:
+            msg = "option {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
-parent_config_parser.add_argument(
+level_group = parent_config_parser.add_mutually_exclusive_group()
+level_group.add_argument(
     "--global",
     dest="level",
     action="store_const",
-    const=Config.LEVEL_GLOBAL,
+    const="global",
     help="Use global config.",
 )
-parent_config_parser.add_argument(
+level_group.add_argument(
     "--system",
     dest="level",
     action="store_const",
-    const=Config.LEVEL_SYSTEM,
+    const="system",
     help="Use system config.",
 )
-parent_config_parser.add_argument(
+level_group.add_argument(
     "--local",
     dest="level",
     action="store_const",
-    const=Config.LEVEL_LOCAL,
+    const="local",
     help="Use local config.",
 )
-parent_config_parser.set_defaults(level=Config.LEVEL_REPO)
+parent_config_parser.set_defaults(level="repo")
 
 
 def add_parser(subparsers, parent_parser):

dvc/command/daemon.py

@@ -1,5 +1,5 @@
-from dvc.command.base import CmdBaseNoRepo
-from dvc.command.base import fix_subparsers
+from dvc.command import completion
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):
@@ -15,11 +15,10 @@ class CmdDaemonUpdater(CmdDaemonBase):
 
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
+        tmp_dir = os.path.join(dvc_dir, "tmp")
         config = Config(dvc_dir, validate=False)
-        hardlink_lock = config.config.get("core", {}).get(
-            "hardlink_lock", False
-        )
-        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
+        hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
+        updater = Updater(tmp_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 
         return 0
@@ -66,5 +65,7 @@ def add_parser(subparsers, parent_parser):
         description=DAEMON_ANALYTICS_HELP,
         help=DAEMON_ANALYTICS_HELP,
     )
-    daemon_analytics_parser.add_argument("target", help="Analytics file.")
+    daemon_analytics_parser.add_argument(
+        "target", help="Analytics file.",
+    ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -0,0 +1,115 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _show_ascii(G):
+    from dvc.repo.graph import get_pipelines
+    from dvc.dagascii import draw
+
+    pipelines = get_pipelines(G)
+
+    ret = []
+    for pipeline in pipelines:
+        ret.append(draw(pipeline.nodes, pipeline.edges))
+
+    return "\n".join(ret)
+
+
+def _show_dot(G):
+    import io
+    from networkx.drawing.nx_pydot import write_dot
+
+    dot_file = io.StringIO()
+    write_dot(G, dot_file)
+    return dot_file.getvalue()
+
+
+def _build(G, target=None, full=False):
+    import networkx as nx
+    from dvc.repo.graph import get_pipeline, get_pipelines
+
+    if target:
+        H = get_pipeline(get_pipelines(G), target)
+        if not full:
+            descendants = nx.descendants(G, target)
+            descendants.add(target)
+            H.remove_nodes_from(set(G.nodes()) - descendants)
+    else:
+        H = G
+
+    def _relabel(stage):
+        return stage.addressing
+
+    return nx.relabel_nodes(H, _relabel, copy=False)
+
+
+class CmdDAG(CmdBase):
+    def run(self):
+        try:
+            target = None
+            if self.args.target:
+                stages = self.repo.collect(self.args.target)
+                if len(stages) > 1:
+                    logger.error(
+                        f"'{self.args.target}' contains more than one stage "
+                        "{stages}, please specify one stage"
+                    )
+                    return 1
+                target = stages[0]
+
+            G = _build(self.repo.graph, target=target, full=self.args.full,)
+
+            if self.args.dot:
+                logger.info(_show_dot(G))
+            else:
+                from dvc.utils.pager import pager
+
+                pager(_show_ascii(G))
+
+            return 0
+        except DvcException:
+            msg = "failed to show "
+            if self.args.target:
+                msg += f"a pipeline for '{target}'"
+            else:
+                msg += "pipelines"
+            logger.exception(msg)
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    DAG_HELP = "Visualize DVC project DAG."
+    dag_parser = subparsers.add_parser(
+        "dag",
+        parents=[parent_parser],
+        description=append_doc_link(DAG_HELP, "dag"),
+        help=DAG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    dag_parser.add_argument(
+        "--dot",
+        action="store_true",
+        default=False,
+        help="Print DAG with .dot format.",
+    )
+    dag_parser.add_argument(
+        "--full",
+        action="store_true",
+        default=False,
+        help=(
+            "Show full DAG that the target belongs too, instead of "
+            "showing DAG consisting only of ancestors."
+        ),
+    )
+    dag_parser.add_argument(
+        "target",
+        nargs="?",
+        help="Stage or output to show pipeline for (optional). "
+        "Finds all stages in the workspace by default.",
+    )
+    dag_parser.set_defaults(func=CmdDAG)

dvc/command/data_sync.py

@@ -1,38 +1,46 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.checkout import log_changes
+from dvc.exceptions import CheckoutError, DvcException
+from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
-    @classmethod
-    def check_up_to_date(cls, processed_files_count):
-        if processed_files_count == 0:
-            logger.info("Everything is up to date.")
+    def log_summary(self, stats):
+        default_msg = "Everything is up to date."
+        logger.info(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
+    def log_summary(self, stats):
+        log_changes(stats)
+        super().log_summary(stats)
+
     def run(self):
         try:
-            processed_files_count = self.repo.pull(
+            stats = self.repo.pull(
                 targets=self.args.targets,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 force=self.args.force,
                 recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
             )
-        except DvcException:
+            self.log_summary(stats)
+        except (CheckoutError, DvcException) as exc:
+            self.log_summary(getattr(exc, "stats", {}))
             logger.exception("failed to pull data from the cloud")
             return 1
-        self.check_up_to_date(processed_files_count)
+
         return 0
 
 
@@ -45,13 +53,15 @@ class CmdDataPush(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
             )
+            self.log_summary({"pushed": processed_files_count})
         except DvcException:
             logger.exception("failed to push data to the cloud")
             return 1
-        self.check_up_to_date(processed_files_count)
         return 0
 
 
@@ -64,13 +74,15 @@ class CmdDataFetch(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
             )
+            self.log_summary({"fetched": processed_files_count})
         except DvcException:
             logger.exception("failed to fetch data from the cloud")
             return 1
-        self.check_up_to_date(processed_files_count)
         return 0
 
 
@@ -78,27 +90,33 @@ def shared_parent_parser():
     from dvc.cli import get_parent_parser
 
     # Parent parser used in pull/push/status
-    shared_parent_parser = argparse.ArgumentParser(
+    parent_parser = argparse.ArgumentParser(
         add_help=False, parents=[get_parent_parser()]
     )
-    shared_parent_parser.add_argument(
-        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
+    parent_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help="Number of jobs to run simultaneously.",
+        metavar="<number>",
     )
-    shared_parent_parser.add_argument(
+    parent_parser.add_argument(
         "targets",
         nargs="*",
-        help="Limit command scope to these DVC-files. "
-        "Using -R, directories to search DVC-files in can also be given.",
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
 
-    return shared_parent_parser
+    return parent_parser
 
 
 def add_parser(subparsers, _parent_parser):
     from dvc.command.status import CmdDataStatus
 
     # Pull
-    PULL_HELP = "Pull data files from a DVC remote storage."
+    PULL_HELP = "Download tracked files or directories from remote storage."
 
     pull_parser = subparsers.add_parser(
         "pull",
@@ -108,7 +126,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
-        "-r", "--remote", help="Remote repository to pull from."
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
     )
     pull_parser.add_argument(
         "-a",
@@ -124,6 +142,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for all tags.",
     )
+    pull_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all commits.",
+    )
     pull_parser.add_argument(
         "-f",
         "--force",
@@ -145,10 +169,16 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Pull cache for subdirectories of the specified directory.",
     )
+    pull_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Fetch run history for all stages.",
+    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    PUSH_HELP = "Push data files to a DVC remote storage."
+    PUSH_HELP = "Upload tracked files or directories to remote storage."
 
     push_parser = subparsers.add_parser(
         "push",
@@ -158,7 +188,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
-        "-r", "--remote", help="Remote repository to push to."
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
     )
     push_parser.add_argument(
         "-a",
@@ -174,6 +204,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Push cache for all tags.",
     )
+    push_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Push cache for all commits.",
+    )
     push_parser.add_argument(
         "-d",
         "--with-deps",
@@ -186,12 +222,21 @@ def add_parser(subparsers, _parent_parser):
         "--recursive",
         action="store_true",
         default=False,
-        help="Push cache from subdirectories of specified directory.",
+        help="Push cache for subdirectories of specified directory.",
+    )
+    push_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Push run history for all stages.",
     )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = "Fetch data files from a DVC remote storage."
+    FETCH_HELP = (
+        "Get tracked files or directories from remote storage"
+        " into the cache."
+    )
 
     fetch_parser = subparsers.add_parser(
         "fetch",
@@ -201,7 +246,10 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
-        "-r", "--remote", help="Remote repository to fetch from."
+        "-r",
+        "--remote",
+        help="Remote storage to fetch from",
+        metavar="<name>",
     )
     fetch_parser.add_argument(
         "-a",
@@ -217,6 +265,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for all tags.",
     )
+    fetch_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all commits.",
+    )
     fetch_parser.add_argument(
         "-d",
         "--with-deps",
@@ -231,6 +285,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for subdirectories of specified directory.",
     )
+    fetch_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Fetch run history for all stages.",
+    )
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
@@ -264,7 +324,10 @@ def add_parser(subparsers, _parent_parser):
         help="Show status of a local cache compared to a remote repository.",
     )
     status_parser.add_argument(
-        "-r", "--remote", help="Remote repository to compare local cache to."
+        "-r",
+        "--remote",
+        help="Remote storage to compare local cache to",
+        metavar="<name>",
     )
     status_parser.add_argument(
         "-a",
@@ -282,6 +345,13 @@ def add_parser(subparsers, _parent_parser):
         help="Show status of a local cache compared to a remote repository "
         "for all tags.",
     )
+    status_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository "
+        "for all commits.",
+    )
     status_parser.add_argument(
         "-d",
         "--with-deps",
@@ -289,4 +359,18 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Show status for all dependencies of the specified target.",
     )
+    status_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Show status of all stages in the specified directory.",
+    )
+    status_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show status in JSON format.",
+    )
+
     status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -2,11 +2,9 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -27,7 +25,7 @@ class CmdDestroy(CmdBase):
                 )
 
             self.repo.destroy()
-        except Exception:
+        except Exception:  # noqa, pylint: disable=broad-except
             logger.exception("failed to destroy DVC")
             return 1
         return 0

dvc/command/diff.py

@@ -1,170 +1,173 @@
 import argparse
+import json
 import logging
+import os
 
-import humanize
-import inflect
-from funcy import compact
+import colorama
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
+def _show_md(diff):
+    from dvc.utils.diff import table
+
+    rows = []
+    for status in ["added", "deleted", "modified"]:
+        entries = diff.get(status, [])
+        if not entries:
+            continue
+        paths = sorted(entry["path"] for entry in entries)
+        for path in paths:
+            rows.append([status, path])
+
+    return table(["Status", "Path"], rows, True)
+
+
 class CmdDiff(CmdBase):
     @staticmethod
-    def _print_size(size):
-        if size < 0:
-            change = "decreased by {}"
-        elif size > 0:
-            change = "increased by {}"
-        else:
-            change = "not changed"
-        natur_size = humanize.naturalsize(abs(size))
-        return change.format(natur_size)
+    def _format(diff):
+        """
+        Given a diff structure, generate a string of paths separated
+        by new lines and grouped together by their state.
 
-    @staticmethod
-    def _get_md5_string(sign, file_name, checksum):
-        sample_msg = ""
-        if file_name:
-            sample_msg = "{}{} with md5 {}\n"
-            sample_msg = sample_msg.format(sign, file_name, checksum)
-        return sample_msg
+        A group's header is colored and its entries are sorted to enhance
+        readability, for example:
 
-    @classmethod
-    def _get_dir_changes(cls, dct):
-        import dvc.repo.diff as diff
+            Added:
+                another_file.txt
+                backup.tar
+                dir/
+                dir/1
 
-        engine = inflect.engine()
-        changes_msg = (
-            "{} {} untouched, {} {} modified, {} {} added, "
-            "{} {} deleted, size was {}"
+        An example of a diff formatted when entries contain hash:
+
+            Added:
+                d3b07384 foo
+
+            Modified:
+                c157a790..f98bf6f1 bar
+
+        If a group has no entries, it won't be included in the result.
+
+        At the bottom, include a summary with the number of files per state.
+        """
+
+        def _digest(checksum):
+            if isinstance(checksum, str):
+                return checksum[0:8]
+            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+        colors = {
+            "added": colorama.Fore.GREEN,
+            "modified": colorama.Fore.YELLOW,
+            "deleted": colorama.Fore.RED,
+        }
+
+        summary = {}
+        groups = []
+
+        for state in ["added", "deleted", "modified"]:
+            summary[state] = 0
+            entries = diff[state]
+
+            if not entries:
+                continue
+
+            content = []
+
+            for entry in entries:
+                path = entry["path"]
+                checksum = entry.get("hash")
+                summary[state] += 1 if not path.endswith(os.sep) else 0
+                content.append(
+                    "{space}{checksum}{separator}{path}".format(
+                        space="    ",
+                        checksum=_digest(checksum) if checksum else "",
+                        separator="  " if checksum else "",
+                        path=entry["path"],
+                    )
+                )
+
+            groups.append(
+                "{color}{header}{nc}:\n{content}".format(
+                    color=colors[state],
+                    header=state.capitalize(),
+                    nc=colorama.Fore.RESET,
+                    content="\n".join(content),
+                )
+            )
+
+        groups.append(
+            "files summary: {added} added, {deleted} deleted,"
+            " {modified} modified".format_map(summary)
         )
-        changes_msg = changes_msg.format(
-            dct[diff.DIFF_IDENT],
-            engine.plural("file", dct[diff.DIFF_IDENT]),
-            dct[diff.DIFF_CHANGE],
-            engine.plural("file", dct[diff.DIFF_CHANGE]),
-            dct[diff.DIFF_NEW],
-            engine.plural("file", dct[diff.DIFF_NEW]),
-            dct[diff.DIFF_DEL],
-            engine.plural("file", dct[diff.DIFF_DEL]),
-            cls._print_size(dct[diff.DIFF_SIZE]),
-        )
-        return changes_msg
 
-    @classmethod
-    def _get_file_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if (
-            dct.get(diff.DIFF_OLD_FILE)
-            and dct.get(diff.DIFF_NEW_FILE)
-            and dct[diff.DIFF_SIZE] == 0
-        ):
-            msg = "file size was not changed"
-        elif dct.get(diff.DIFF_NEW_FILE):
-            msg = "added file with size {}".format(
-                humanize.naturalsize(dct[diff.DIFF_SIZE])
-            )
-        elif dct.get(diff.DIFF_OLD_FILE):
-            msg = "deleted file with size {}".format(
-                humanize.naturalsize(abs(dct[diff.DIFF_SIZE]))
-            )
-        else:
-            msg = "file was modified, file size {}".format(
-                cls._print_size(dct[diff.DIFF_SIZE])
-            )
-        return msg
-
-    @classmethod
-    def _get_royal_changes(cls, dct):
-        import dvc.repo.diff as diff
-
-        if dct[diff.DIFF_SIZE] != diff.DIFF_SIZE_UNKNOWN:
-            if dct.get("is_dir"):
-                return cls._get_dir_changes(dct)
-            else:
-                return cls._get_file_changes(dct)
-        return "size is ?"
-
-    @classmethod
-    def _show(cls, diff_dct):
-        import dvc.repo.diff as diff
-
-        msg = "dvc diff from {} to {}".format(
-            diff_dct[diff.DIFF_A_REF], diff_dct[diff.DIFF_B_REF]
-        )
-        if diff_dct.get(diff.DIFF_EQUAL):
-            logger.info(msg)
-            return
-        for dct in diff_dct[diff.DIFF_LIST]:
-            msg += "\n\ndiff for '{}'\n".format(dct[diff.DIFF_TARGET])
-            msg += cls._get_md5_string(
-                "-",
-                dct.get(diff.DIFF_OLD_FILE),
-                dct.get(diff.DIFF_OLD_CHECKSUM),
-            )
-            msg += cls._get_md5_string(
-                "+",
-                dct.get(diff.DIFF_NEW_FILE),
-                dct.get(diff.DIFF_NEW_CHECKSUM),
-            )
-            msg += "\n"
-            msg += cls._get_royal_changes(dct)
-        logger.info(msg)
-        return msg
+        return "\n\n".join(groups)
 
     def run(self):
         try:
-            msg = self.repo.diff(
-                self.args.a_ref, target=self.args.target, b_ref=self.args.b_ref
-            )
-            self._show(msg)
+            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+
+            if not self.args.show_hash:
+                for _, entries in diff.items():
+                    for entry in entries:
+                        del entry["hash"]
+
+            if self.args.show_json:
+                logger.info(json.dumps(diff))
+            elif self.args.show_md:
+                logger.info(_show_md(diff))
+            elif diff:
+                logger.info(self._format(diff))
+
         except DvcException:
-            msg = "failed to get 'diff {}'"
-            args = " ".join(
-                compact([self.args.target, self.args.a_ref, self.args.b_ref])
-            )
-            msg = msg.format(args)
-            logger.exception(msg)
+            logger.exception("failed to get diff")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     DIFF_DESCRIPTION = (
-        "Show diff of a data file or a directory that is under DVC control.\n"
-        "Some basic statistics summary, how many files were deleted/changed."
+        "Show added, modified, or deleted data between commits in the DVC"
+        " repository, or between a commit and the workspace."
     )
-    DIFF_HELP = "Show a diff of a DVC controlled data file or a directory."
     diff_parser = subparsers.add_parser(
         "diff",
         parents=[parent_parser],
         description=append_doc_link(DIFF_DESCRIPTION, "diff"),
-        help=DIFF_HELP,
+        help=DIFF_DESCRIPTION,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     diff_parser.add_argument(
-        "-t",
-        "--target",
-        help=(
-            "Source path to a data file or directory. Default None. "
-            "If not specified, compares all files and directories "
-            "that are under DVC control in the current working space."
-        ),
+        "a_rev",
+        help="Old Git commit to compare (defaults to HEAD)",
+        nargs="?",
+        default="HEAD",
     )
     diff_parser.add_argument(
-        "a_ref", help="Git reference from which diff calculates"
-    )
-    diff_parser.add_argument(
-        "b_ref",
-        help=(
-            "Git reference until which diff calculates, if omitted "
-            "diff shows the difference between current HEAD and a_ref"
-        ),
+        "b_rev",
+        help=("New Git commit to compare (defaults to the current workspace)"),
         nargs="?",
     )
+    diff_parser.add_argument(
+        "--show-json",
+        help="Format the output into a JSON",
+        action="store_true",
+        default=False,
+    )
+    diff_parser.add_argument(
+        "--show-hash",
+        help="Display hash value for each entry",
+        action="store_true",
+        default=False,
+    )
+    diff_parser.add_argument(
+        "--show-md",
+        help="Show tabulated output in the Markdown format (GFM).",
+        action="store_true",
+        default=False,
+    )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/experiments.py

@@ -0,0 +1,360 @@
+import argparse
+import io
+import logging
+from collections import OrderedDict
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _update_names(names, items):
+    from flatten_json import flatten
+
+    for name, item in items:
+        if isinstance(item, dict):
+            item = flatten(item, ".")
+            names.update(item.keys())
+        else:
+            names.add(name)
+
+
+def _collect_names(all_experiments):
+    metric_names = set()
+    param_names = set()
+
+    for _, experiments in all_experiments.items():
+        for exp in experiments.values():
+            _update_names(metric_names, exp.get("metrics", {}).items())
+            _update_names(param_names, exp.get("params", {}).items())
+
+    return sorted(metric_names), sorted(param_names)
+
+
+def _collect_rows(
+    base_rev, experiments, metric_names, param_names, precision=None
+):
+    from flatten_json import flatten
+
+    if precision is None:
+        precision = DEFAULT_PRECISION
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    def _extend(row, names, items):
+        if not items:
+            row.extend(["-"] * len(names))
+            return
+
+        for fname, item in items:
+            if isinstance(item, dict):
+                item = flatten(item, ".")
+            else:
+                item = {fname: item}
+            for name in names:
+                if name in item:
+                    row.append(str(_round(item[name])))
+                else:
+                    row.append("-")
+
+    for i, (rev, exp) in enumerate(experiments.items()):
+        row = []
+        style = None
+        queued = "*" if exp.get("queued", False) else ""
+        if rev == "baseline":
+            row.append(f"{base_rev}")
+            style = "bold"
+        elif i < len(experiments) - 1:
+            row.append(f"├── {queued}{rev[:7]}")
+        else:
+            row.append(f"└── {queued}{rev[:7]}")
+
+        _extend(row, metric_names, exp.get("metrics", {}).items())
+        _extend(row, param_names, exp.get("params", {}).items())
+
+        yield row, style
+
+
+def _show_experiments(all_experiments, console, precision=None):
+    from rich.table import Table
+    from dvc.scm.git import Git
+
+    metric_names, param_names = _collect_names(all_experiments)
+
+    table = Table()
+    table.add_column("Experiment")
+    for name in metric_names:
+        table.add_column(name, justify="right")
+    for name in param_names:
+        table.add_column(name, justify="left")
+
+    for base_rev, experiments in all_experiments.items():
+        if Git.is_sha(base_rev):
+            base_rev = base_rev[:7]
+
+        for row, _, in _collect_rows(
+            base_rev,
+            experiments,
+            metric_names,
+            param_names,
+            precision=precision,
+        ):
+            table.add_row(*row)
+
+    console.print(table)
+
+
+class CmdExperimentsShow(CmdBase):
+    def run(self):
+        from rich.console import Console
+        from dvc.utils.pager import pager
+
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            all_experiments = self.repo.experiments.show(
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+            )
+
+            # Note: rich does not currently include a native way to force
+            # infinite width for use with a pager
+            console = Console(
+                file=io.StringIO(), force_terminal=True, width=9999
+            )
+
+            _show_experiments(all_experiments, console)
+
+            pager(console.file.getvalue())
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
+
+        return 0
+
+
+class CmdExperimentsCheckout(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        self.repo.experiments.checkout(
+            self.args.experiment, force=self.args.force
+        )
+
+        return 0
+
+
+def _show_diff(
+    diff, title="", markdown=False, no_path=False, old=False, precision=None
+):
+    from dvc.utils.diff import table
+
+    if precision is None:
+        precision = DEFAULT_PRECISION
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    rows = []
+    for fname, diff_ in diff.items():
+        sorted_diff = OrderedDict(sorted(diff_.items()))
+        for item, change in sorted_diff.items():
+            row = [] if no_path else [fname]
+            row.append(item)
+            if old:
+                row.append(_round(change.get("old")))
+            row.append(_round(change["new"]))
+            row.append(_round(change.get("diff", "diff not supported")))
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append(title)
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
+
+
+class CmdExperimentsDiff(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            diff = self.repo.experiments.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                diffs = [("metrics", "Metric"), ("params", "Param")]
+                for key, title in diffs:
+                    table = _show_diff(
+                        diff[key],
+                        title=title,
+                        markdown=self.args.show_md,
+                        no_path=self.args.no_path,
+                        old=self.args.old,
+                        precision=self.args.precision,
+                    )
+                    if table:
+                        logger.info(table)
+                        logger.info("")
+
+        except DvcException:
+            logger.exception("failed to show experiments diff")
+            return 1
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    EXPERIMENTS_HELP = "Commands to display and compare experiments."
+
+    experiments_parser = subparsers.add_parser(
+        "experiments",
+        parents=[parent_parser],
+        aliases=["exp"],
+        description=append_doc_link(EXPERIMENTS_HELP, "experiments"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    experiments_subparsers = experiments_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc experiments CMD --help` to display "
+        "command-specific help.",
+    )
+
+    fix_subparsers(experiments_subparsers)
+
+    EXPERIMENTS_SHOW_HELP = "Print experiments."
+    experiments_show_parser = experiments_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "experiments/show"),
+        help=EXPERIMENTS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show metrics for all branches.",
+    )
+    experiments_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show metrics for all tags.",
+    )
+    experiments_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
+    experiments_show_parser.set_defaults(func=CmdExperimentsShow)
+
+    EXPERIMENTS_CHECKOUT_HELP = "Checkout experiments."
+    experiments_checkout_parser = experiments_subparsers.add_parser(
+        "checkout",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_CHECKOUT_HELP, "experiments/checkout"
+        ),
+        help=EXPERIMENTS_CHECKOUT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_checkout_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite your current workspace with changes from the "
+        "experiment.",
+    )
+    experiments_checkout_parser.add_argument(
+        "experiment", help="Checkout this experiment.",
+    )
+    experiments_checkout_parser.set_defaults(func=CmdExperimentsCheckout)
+
+    EXPERIMENTS_DIFF_HELP = (
+        "Show changes between experiments in the DVC repository."
+    )
+    experiments_diff_parser = experiments_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "experiments/diff"),
+        help=EXPERIMENTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
+    )
+    experiments_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help="New experiment to compare (defaults to the current workspace)",
+    )
+    experiments_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics/params as well.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric/param value.",
+    )
+    experiments_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric/param path.",
+    )
+    experiments_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)

dvc/command/freeze.py

@@ -0,0 +1,58 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdFreezeBase(CmdBase):
+    def _run(self, func, name):
+        ret = 0
+        for target in self.args.targets:
+            try:
+                func(target)
+            except DvcException:
+                logger.exception(f"failed to {name} '{target}'")
+                ret = 1
+        return ret
+
+
+class CmdFreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.freeze, "freeze")
+
+
+class CmdUnfreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.unfreeze, "unfreeze")
+
+
+def add_parser(subparsers, parent_parser):
+    FREEZE_HELP = "Freeze stages or .dvc files."
+    freeze_parser = subparsers.add_parser(
+        "freeze",
+        parents=[parent_parser],
+        description=append_doc_link(FREEZE_HELP, "freeze"),
+        help=FREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    freeze_parser.add_argument(
+        "targets", nargs="+", help="Stages or .dvc files to freeze",
+    ).complete = completion.DVC_FILE
+    freeze_parser.set_defaults(func=CmdFreeze)
+
+    UNFREEZE_HELP = "Unfreeze stages or .dvc files."
+    unfreeze_parser = subparsers.add_parser(
+        "unfreeze",
+        parents=[parent_parser],
+        description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
+        help=UNFREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    unfreeze_parser.add_argument(
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+    ).complete = completion.DVC_FILE
+    unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -3,18 +3,25 @@ import logging
 import os
 
 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-
+from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)
 
 
 class CmdGC(CmdBase):
     def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
         msg = "This will remove all cache except items used in "
 
-        msg += "the working tree"
+        msg += "the workspace"
         if self.args.all_commits:
             msg += " and all git commits"
         elif self.args.all_branches and self.args.all_tags:
@@ -47,15 +54,16 @@ class CmdGC(CmdBase):
             force=self.args.force,
             jobs=self.args.jobs,
             repos=self.args.repos,
+            workspace=self.args.workspace,
         )
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    GC_HELP = "Collect unused data from DVC cache or a remote storage."
+    GC_HELP = "Garbage collect unused objects from cache or remote storage."
     GC_DESCRIPTION = (
-        "Deletes all files in the cache or a remote which are not in\n"
-        "use by the specified git references (defaults to just HEAD)."
+        "Removes all files in the cache or a remote which are not in\n"
+        "use by the specified Git revisions (defaults to just HEAD)."
     )
     gc_parser = subparsers.add_parser(
         "gc",
@@ -64,25 +72,32 @@ def add_parser(subparsers, parent_parser):
         help=GC_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep data files used in the current workspace.",
+    )
     gc_parser.add_argument(
         "-a",
         "--all-branches",
         action="store_true",
         default=False,
-        help="Keep data files for the tips of all git branches.",
+        help="Keep data files for the tips of all Git branches.",
     )
     gc_parser.add_argument(
         "-T",
         "--all-tags",
         action="store_true",
         default=False,
-        help="Keep data files for all git tags.",
+        help="Keep data files for all Git tags.",
     )
     gc_parser.add_argument(
         "--all-commits",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help="Keep data files for all Git commits.",
     )
     gc_parser.add_argument(
         "-c",
@@ -92,7 +107,10 @@ def add_parser(subparsers, parent_parser):
         help="Collect garbage in remote repository.",
     )
     gc_parser.add_argument(
-        "-r", "--remote", help="Remote storage to collect garbage in."
+        "-r",
+        "--remote",
+        help="Remote storage to collect garbage in",
+        metavar="<name>",
     )
     gc_parser.add_argument(
         "-f",
@@ -102,7 +120,11 @@ def add_parser(subparsers, parent_parser):
         help="Force garbage collection - automatically agree to all prompts.",
     )
     gc_parser.add_argument(
-        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
+        "-j",
+        "--jobs",
+        type=int,
+        help="Number of jobs to run simultaneously.",
+        metavar="<number>",
     )
     gc_parser.add_argument(
         "-p",
@@ -113,5 +135,6 @@ def add_parser(subparsers, parent_parser):
         help="Keep data files required by these projects "
         "in addition to the current one. "
         "Useful if you share a single cache across repos.",
+        metavar="<paths>",
     )
     gc_parser.set_defaults(func=CmdGC)

dvc/command/get.py

@@ -1,10 +1,10 @@
 import argparse
 import logging
 
-from .base import append_doc_link
-from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
+from . import completion
+from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -51,9 +51,7 @@ class CmdGet(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    GET_HELP = (
-        "Download file or directory from any DVC project or Git repository."
-    )
+    GET_HELP = "Download file or directory tracked by DVC or by Git."
     get_parser = subparsers.add_parser(
         "get",
         parents=[parent_parser],
@@ -62,18 +60,23 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     get_parser.add_argument(
-        "url",
-        help="Location of DVC project or Git repository to download from",
+        "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the project or repository",
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     get_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to download files to"
-    )
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
+    ).complete = completion.DIR
     get_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
     )
     get_parser.add_argument(
         "--show-url",

dvc/command/get_url.py

@@ -1,10 +1,10 @@
 import argparse
 import logging
 
-from .base import append_doc_link
-from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
+from . import completion
+from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -17,7 +17,7 @@ class CmdGetUrl(CmdBaseNoRepo):
             Repo.get_url(self.args.url, out=self.args.out)
             return 0
         except DvcException:
-            logger.exception("failed to get '{}'".format(self.args.url))
+            logger.exception(f"failed to get '{self.args.url}'")
             return 1
 
 
@@ -34,6 +34,6 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to."
-    )
+        "out", nargs="?", help="Destination path to put data to.",
+    ).complete = completion.DIR
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -0,0 +1,115 @@
+import logging
+import os
+
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.exceptions import NotDvcRepoError
+
+logger = logging.getLogger(__name__)
+
+
+class CmdHookBase(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            repo = Repo()
+            repo.close()
+        except NotDvcRepoError:
+            return 0
+
+        return self._run()
+
+    def _run(self):
+        raise NotImplementedError
+
+
+class CmdPreCommit(CmdHookBase):
+    def _run(self):
+        from dvc.main import main
+
+        return main(["status"])
+
+
+class CmdPostCheckout(CmdHookBase):
+    def _run(self):
+        # when we are running from pre-commit tool, it doesn't provide CLI
+        # flags, but instead provides respective env vars that we could use.
+        flag = os.environ.get("PRE_COMMIT_CHECKOUT_TYPE")
+        if flag is None and len(self.args.args) >= 3:
+            # see https://git-scm.com/docs/githooks#_post_checkout
+            flag = self.args.args[2]
+
+        # checking out some reference and not specific file.
+        if flag != "1":
+            return 0
+
+        # make sure we are not in the middle of a rebase/merge, so we
+        # don't accidentally break it with an unsuccessful checkout.
+        # Note that git hooks are always running in repo root.
+        if os.path.isdir(os.path.join(".git", "rebase-merge")):
+            return 0
+
+        from dvc.main import main
+
+        return main(["checkout"])
+
+
+class CmdPrePush(CmdHookBase):
+    def _run(self):
+        from dvc.main import main
+
+        return main(["push"])
+
+
+def add_parser(subparsers, parent_parser):
+    GIT_HOOK_HELP = "Run GIT hook."
+
+    git_hook_parser = subparsers.add_parser(
+        "git-hook",
+        parents=[parent_parser],
+        description=GIT_HOOK_HELP,
+        add_help=False,
+    )
+
+    git_hook_subparsers = git_hook_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific help.",
+    )
+
+    fix_subparsers(git_hook_subparsers)
+
+    PRE_COMMIT_HELP = "Run pre-commit GIT hook."
+    pre_commit_parser = git_hook_subparsers.add_parser(
+        "pre-commit",
+        parents=[parent_parser],
+        description=PRE_COMMIT_HELP,
+        help=PRE_COMMIT_HELP,
+    )
+    pre_commit_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_commit_parser.set_defaults(func=CmdPreCommit)
+
+    POST_CHECKOUT_HELP = "Run post-checkout GIT hook."
+    post_checkout_parser = git_hook_subparsers.add_parser(
+        "post-checkout",
+        parents=[parent_parser],
+        description=POST_CHECKOUT_HELP,
+        help=POST_CHECKOUT_HELP,
+    )
+    post_checkout_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    post_checkout_parser.set_defaults(func=CmdPostCheckout)
+
+    PRE_PUSH_HELP = "Run pre-push GIT hook."
+    pre_push_parser = git_hook_subparsers.add_parser(
+        "pre-push",
+        parents=[parent_parser],
+        description=PRE_PUSH_HELP,
+        help=PRE_PUSH_HELP,
+    )
+    pre_push_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_push_parser.set_defaults(func=CmdPrePush)

dvc/command/imp.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -30,7 +29,7 @@ class CmdImport(CmdBase):
 
 def add_parser(subparsers, parent_parser):
     IMPORT_HELP = (
-        "Download file or directory from any DVC project or Git repository "
+        "Download file or directory tracked by DVC or by Git "
         "into the workspace, and track it."
     )
 
@@ -42,17 +41,22 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawTextHelpFormatter,
     )
     import_parser.add_argument(
-        "url",
-        help="Location of DVC project or Git repository to download from",
+        "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path",
-        help="Path to a file or directory within the project or repository",
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     import_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to download files to"
-    )
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
+    ).complete = completion.DIR
     import_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -13,7 +12,10 @@ class CmdImportUrl(CmdBase):
     def run(self):
         try:
             self.repo.imp_url(
-                self.args.url, out=self.args.out, fname=self.args.file
+                self.args.url,
+                out=self.args.out,
+                fname=self.args.file,
+                no_exec=self.args.no_exec,
             )
         except DvcException:
             logger.exception(
@@ -56,9 +58,17 @@ def add_parser(subparsers, parent_parser):
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to."
-    )
+        "out", nargs="?", help="Destination path to put files to.",
+    ).complete = completion.DIR
     import_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
+    ).complete = completion.DIR
+    import_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create DVC-file without actually downloading it.",
     )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/init.py

@@ -1,9 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -15,7 +13,10 @@ class CmdInit(CmdBaseNoRepo):
 
         try:
             self.repo = Repo.init(
-                ".", no_scm=self.args.no_scm, force=self.args.force
+                ".",
+                no_scm=self.args.no_scm,
+                force=self.args.force,
+                subdir=self.args.subdir,
             )
             self.config = self.repo.config
         except InitError:
@@ -44,7 +45,7 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help="Initiate DVC in directory that is "
-        "not tracked by any scm tool (e.g. git).",
+        "not tracked by any SCM tool (e.g. Git).",
     )
     init_parser.add_argument(
         "-f",
@@ -56,4 +57,13 @@ def add_parser(subparsers, parent_parser):
             "This operation removes local cache."
         ),
     )
+    init_parser.add_argument(
+        "--subdir",
+        action="store_true",
+        default=False,
+        help=(
+            "Necessary for running this command inside a subdirectory of a "
+            "parent SCM repository."
+        ),
+    )
     init_parser.set_defaults(func=CmdInit)

dvc/command/install.py

@@ -1,9 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-
+from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -11,8 +9,8 @@ logger = logging.getLogger(__name__)
 class CmdInstall(CmdBase):
     def run(self):
         try:
-            self.repo.install()
-        except Exception:
+            self.repo.install(self.args.use_pre_commit_tool)
+        except Exception:  # noqa, pylint:disable=broad-except
             logger.exception("failed to install DVC Git hooks")
             return 1
         return 0
@@ -27,4 +25,11 @@ def add_parser(subparsers, parent_parser):
         help=INSTALL_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    install_parser.add_argument(
+        "--use-pre-commit-tool",
+        action="store_true",
+        default=False,
+        help="Install DVC hooks using pre-commit "
+        "(https://pre-commit.com) if it is installed.",
+    )
     install_parser.set_defaults(func=CmdInstall)

dvc/command/lock.py

@@ -1,61 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdLockBase(CmdBase):
-    def _run(self, unlock):
-        for target in self.args.targets:
-            try:
-                self.repo.lock_stage(target, unlock=unlock)
-            except DvcException:
-                logger.exception(
-                    "failed to {}lock '{}'".format(
-                        "un" if unlock else "", target
-                    )
-                )
-
-                return 1
-        return 0
-
-
-class CmdLock(CmdLockBase):
-    def run(self):
-        return self._run(False)
-
-
-class CmdUnlock(CmdLockBase):
-    def run(self):
-        return self._run(True)
-
-
-def add_parser(subparsers, parent_parser):
-    LOCK_HELP = "Lock DVC-files."
-    lock_parser = subparsers.add_parser(
-        "lock",
-        parents=[parent_parser],
-        description=append_doc_link(LOCK_HELP, "lock"),
-        help=LOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    lock_parser.add_argument("targets", nargs="+", help="DVC-files to lock.")
-    lock_parser.set_defaults(func=CmdLock)
-
-    UNLOCK_HELP = "Unlock DVC-files."
-    unlock_parser = subparsers.add_parser(
-        "unlock",
-        parents=[parent_parser],
-        description=append_doc_link(UNLOCK_HELP, "unlock"),
-        help=UNLOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    unlock_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to unlock."
-    )
-    unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/ls/__init__.py

@@ -0,0 +1,79 @@
+import argparse
+import logging
+import sys
+
+from dvc.command import completion
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.ls.ls_colors import LsColors
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _prettify(entries, with_color=False):
+    if with_color:
+        ls_colors = LsColors()
+        fmt = ls_colors.format
+    else:
+
+        def fmt(entry):
+            return entry["path"]
+
+    return [fmt(entry) for entry in entries]
+
+
+class CmdList(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            entries = Repo.ls(
+                self.args.url,
+                self.args.path,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+                dvc_only=self.args.dvc_only,
+            )
+            if entries:
+                entries = _prettify(entries, sys.stdout.isatty())
+                logger.info("\n".join(entries))
+            return 0
+        except DvcException:
+            logger.exception(f"failed to list '{self.args.url}'")
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    LIST_HELP = (
+        "List repository contents, including files"
+        " and directories tracked by DVC and by Git."
+    )
+    list_parser = subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(LIST_HELP, "list"),
+        help=LIST_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    list_parser.add_argument("url", help="Location of DVC repository to list")
+    list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    list_parser.add_argument(
+        "--dvc-only", action="store_true", help="Show only DVC outputs."
+    )
+    list_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    list_parser.add_argument(
+        "path",
+        nargs="?",
+        help="Path to directory within the repository to list outputs for",
+    ).complete = completion.DIR
+    list_parser.set_defaults(func=CmdList)

dvc/command/ls/ls_colors.py

@@ -0,0 +1,48 @@
+import os
+
+
+class LsColors:
+    default = "rs=0:di=01;34:ex=01;32"
+
+    def __init__(self, lscolors=None):
+        self._extensions = {}
+        self._codes = {}
+        self._load(lscolors or os.environ.get("LS_COLORS") or LsColors.default)
+
+    def _load(self, lscolors):
+        for item in lscolors.split(":"):
+            try:
+                code, color = item.split("=", 1)
+            except ValueError:
+                continue
+            if code.startswith("*."):
+                self._extensions[code[1:]] = color
+            else:
+                self._codes[code] = color
+
+    def format(self, entry):
+        text = entry["path"]
+
+        if entry.get("isout", False) and "out" in self._codes:
+            return self._format(text, code="out")
+
+        if entry.get("isdir", False):
+            return self._format(text, code="di")
+
+        if entry.get("isexec", False):
+            return self._format(text, code="ex")
+
+        _, ext = os.path.splitext(text)
+        return self._format(text, ext=ext)
+
+    def _format(self, text, code=None, ext=None):
+        val = None
+        if ext:
+            val = self._extensions.get(ext, None)
+        if code:
+            val = self._codes.get(code, None)
+
+        if not val:
+            return text
+        rs = self._codes.get("rs", 0)
+        return f"\033[{val}m{text}\033[{rs}m"

dvc/command/metrics.py

@@ -1,46 +1,38 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import BadMetricError
-from dvc.exceptions import DvcException
-
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import BadMetricError, DvcException
 
 logger = logging.getLogger(__name__)
 
 
-def show_metrics(metrics, all_branches=False, all_tags=False):
-    """
-    Args:
-        metrics (list): Where each element is either a `list`
-            if an xpath was specified, otherwise a `str`
-    """
+DEFAULT_PRECISION = 5
+
+
+def show_metrics(
+    metrics, all_branches=False, all_tags=False, all_commits=False
+):
+    from flatten_json import flatten
+    from dvc.utils.diff import format_dict
+
     # When `metrics` contains a `None` key, it means that some files
     # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
     missing = metrics.pop(None, None)
 
     for branch, val in metrics.items():
-        if all_branches or all_tags:
-            logger.info("{branch}:".format(branch=branch))
+        if all_branches or all_tags or all_commits:
+            logger.info(f"{branch}:")
 
         for fname, metric in val.items():
-            if isinstance(metric, dict):
-                lines = list(metric.values())
-            elif isinstance(metric, list):
-                lines = metric
-            else:
-                lines = metric.splitlines()
+            if not isinstance(metric, dict):
+                logger.info("\t{}: {}".format(fname, str(metric)))
+                continue
 
-            if len(lines) > 1:
-                logger.info("\t{fname}:".format(fname=fname))
-
-                for line in lines:
-                    logger.info("\t\t{content}".format(content=line))
-
-            else:
-                logger.info("\t{}: {}".format(fname, metric))
+            logger.info(f"\t{fname}:")
+            for key, value in flatten(format_dict(metric), ".").items():
+                logger.info(f"\t\t{key}: {value}")
 
     if missing:
         raise BadMetricError(missing)
@@ -51,14 +43,23 @@ class CmdMetricsShow(CmdBase):
         try:
             metrics = self.repo.metrics.show(
                 self.args.targets,
-                typ=self.args.type,
-                xpath=self.args.xpath,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 recursive=self.args.recursive,
             )
 
-            show_metrics(metrics, self.args.all_branches, self.args.all_tags)
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(metrics))
+            else:
+                show_metrics(
+                    metrics,
+                    self.args.all_branches,
+                    self.args.all_tags,
+                    self.args.all_commits,
+                )
         except DvcException:
             logger.exception("failed to show metrics")
             return 1
@@ -66,82 +67,52 @@ class CmdMetricsShow(CmdBase):
         return 0
 
 
-class CmdMetricsModify(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.modify(
-                self.args.path, typ=self.args.type, xpath=self.args.xpath
-            )
-        except DvcException:
-            logger.exception("failed to modify metric file settings")
-            return 1
+def _show_diff(diff, markdown=False, no_path=False, old=False, precision=None):
+    from collections import OrderedDict
 
-        return 0
+    from dvc.utils.diff import table
 
+    if precision is None:
+        precision = DEFAULT_PRECISION
 
-class CmdMetricsAdd(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.add(
-                self.args.path, self.args.type, self.args.xpath
-            )
-        except DvcException:
-            msg = "failed to add metric file '{}'".format(self.args.path)
-            logger.exception(msg)
-            return 1
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
 
-        return 0
+        return val
 
-
-class CmdMetricsRemove(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.remove(self.args.path)
-        except DvcException:
-            msg = "failed to remove metric file '{}'".format(self.args.path)
-            logger.exception(msg)
-            return 1
-
-        return 0
-
-
-def _show_diff(diff):
-    from texttable import Texttable
-
-    if not diff:
-        return "No changes."
-
-    table = Texttable()
-
-    # remove borders to make it easier for users to copy stuff
-    table.set_chars(("", "", "", ""))
-    table.set_deco(0)
-
-    rows = [["Path", "Metric", "Value", "Change"]]
+    rows = []
     for fname, mdiff in diff.items():
-        for metric, change in mdiff.items():
-            rows.append(
-                [
-                    fname,
-                    metric,
-                    change["new"],
-                    change.get("diff", "diff not supported"),
-                ]
-            )
-    table.add_rows(rows)
-    return table.draw()
+        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
+        for metric, change in sorted_mdiff.items():
+            row = [] if no_path else [fname]
+            row.append(metric)
+            if old:
+                row.append(_round(change.get("old")))
+            row.append(_round(change["new"]))
+            row.append(_round(change.get("diff", "diff not supported")))
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append("Metric")
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
 
 
 class CmdMetricsDiff(CmdBase):
     def run(self):
         try:
             diff = self.repo.metrics.diff(
-                a_ref=self.args.a_ref,
-                b_ref=self.args.b_ref,
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
                 targets=self.args.targets,
-                typ=self.args.type,
-                xpath=self.args.xpath,
                 recursive=self.args.recursive,
+                all=self.args.all,
             )
 
             if self.args.show_json:
@@ -149,7 +120,15 @@ class CmdMetricsDiff(CmdBase):
 
                 logger.info(json.dumps(diff))
             else:
-                logger.info(_show_diff(diff))
+                table = _show_diff(
+                    diff,
+                    self.args.show_md,
+                    self.args.no_path,
+                    self.args.old,
+                    precision=self.args.precision,
+                )
+                if table:
+                    logger.info(table)
 
         except DvcException:
             logger.exception("failed to show metrics diff")
@@ -159,7 +138,7 @@ class CmdMetricsDiff(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Commands to add, manage, collect and display metrics."
+    METRICS_HELP = "Commands to display and compare metrics."
 
     metrics_parser = subparsers.add_parser(
         "metrics",
@@ -176,7 +155,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(metrics_subparsers)
 
-    METRICS_SHOW_HELP = "Output metric values."
+    METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
@@ -187,21 +166,11 @@ def add_parser(subparsers, parent_parser):
     metrics_show_parser.add_argument(
         "targets",
         nargs="*",
-        help="Metric files or directories (see -R) to show "
-        "(leave empty to display all)",
-    )
-    metrics_show_parser.add_argument(
-        "-t",
-        "--type",
         help=(
-            "Type of metrics (json/tsv/htsv/csv/hcsv). "
-            "It can be detected by the file extension automatically. "
-            "Unsupported types will be treated as raw."
+            "Limit command scope to these metric files. Using -R, "
+            "directories to search metric files in can also be given."
         ),
-    )
-    metrics_show_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
+    ).complete = completion.FILE
     metrics_show_parser.add_argument(
         "-a",
         "--all-branches",
@@ -216,6 +185,18 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show metrics for all tags.",
     )
+    metrics_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
+    metrics_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -228,52 +209,10 @@ def add_parser(subparsers, parent_parser):
     )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
-    METRICS_ADD_HELP = "Tag file as a metric file."
-    metrics_add_parser = metrics_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
-        help=METRICS_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
+    METRICS_DIFF_HELP = (
+        "Show changes in metrics between commits in the DVC repository, or "
+        "between a commit and the workspace."
     )
-    metrics_add_parser.add_argument(
-        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
-    )
-    metrics_add_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
-    metrics_add_parser.add_argument("path", help="Path to a metric file.")
-    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
-
-    METRICS_MODIFY_HELP = "Modify metric file options."
-    metrics_modify_parser = metrics_subparsers.add_parser(
-        "modify",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_MODIFY_HELP, "metrics/modify"),
-        help=METRICS_MODIFY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_modify_parser.add_argument(
-        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
-    )
-    metrics_modify_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
-    metrics_modify_parser.add_argument("path", help="Path to a metric file.")
-    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
-
-    METRICS_REMOVE_HELP = "Remove files's metric tag."
-    metrics_remove_parser = metrics_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
-        help=METRICS_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
-    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
-
-    METRICS_DIFF_HELP = "Output metric values."
     metrics_diff_parser = metrics_subparsers.add_parser(
         "diff",
         parents=[parent_parser],
@@ -282,41 +221,22 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     metrics_diff_parser.add_argument(
-        "a_ref",
-        nargs="?",
-        help=(
-            "Git reference from which diff is calculated. "
-            "If omitted `HEAD`(latest commit) is used."
-        ),
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
     )
     metrics_diff_parser.add_argument(
-        "b_ref",
+        "b_rev",
         nargs="?",
-        help=(
-            "Git reference to which diff is calculated. "
-            "If omitted current working tree is used."
-        ),
+        help="New Git commit to compare (defaults to the current workspace)",
     )
     metrics_diff_parser.add_argument(
         "--targets",
         nargs="*",
         help=(
-            "Metric files or directories (see -R) to show diff for. "
-            "Shows diff for all metric files by default."
+            "Limit command scope to these metric files. Using -R, "
+            "directories to search metric files in can also be given."
         ),
-    )
-    metrics_diff_parser.add_argument(
-        "-t",
-        "--type",
-        help=(
-            "Type of metrics (json/tsv/htsv/csv/hcsv). "
-            "It can be detected by the file extension automatically. "
-            "Unsupported types will be treated as raw."
-        ),
-    )
-    metrics_diff_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
+        metavar="<paths>",
+    ).complete = completion.FILE
     metrics_diff_parser.add_argument(
         "-R",
         "--recursive",
@@ -327,10 +247,43 @@ def add_parser(subparsers, parent_parser):
             "metric files."
         ),
     )
+    metrics_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics as well.",
+    )
     metrics_diff_parser.add_argument(
         "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
+    metrics_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    metrics_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric value.",
+    )
+    metrics_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric path.",
+    )
+    metrics_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics to `n` digits precision after the decimal point. "
+            f"Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
     metrics_diff_parser.set_defaults(func=CmdMetricsDiff)

dvc/command/move.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -38,7 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory."
-    )
-    move_parser.add_argument("dst", help="Destination path.")
+        "src", help="Source path to a data file or directory.",
+    ).complete = completion.FILE
+    move_parser.add_argument(
+        "dst", help="Destination path.",
+    ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/params.py

@@ -0,0 +1,118 @@
+import argparse
+import logging
+from collections import OrderedDict
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _show_diff(diff, markdown=False, no_path=False):
+    from dvc.utils.diff import table
+
+    rows = []
+    for fname, pdiff in diff.items():
+        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
+        for param, change in sorted_pdiff.items():
+            row = [] if no_path else [fname]
+            row.append(param)
+            row.append(change["old"])
+            row.append(change["new"])
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append("Param")
+    header.append("Old")
+    header.append("New")
+
+    return table(header, rows, markdown)
+
+
+class CmdParamsDiff(CmdBase):
+    def run(self):
+        try:
+            diff = self.repo.params.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                table = _show_diff(diff, self.args.show_md, self.args.no_path)
+                if table:
+                    logger.info(table)
+
+        except DvcException:
+            logger.exception("failed to show params diff")
+            return 1
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PARAMS_HELP = "Commands to display params."
+
+    params_parser = subparsers.add_parser(
+        "params",
+        parents=[parent_parser],
+        description=append_doc_link(PARAMS_HELP, "params"),
+        help=PARAMS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    params_subparsers = params_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc params CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(params_subparsers)
+
+    PARAMS_DIFF_HELP = (
+        "Show changes in params between commits in the DVC repository, or "
+        "between a commit and the workspace."
+    )
+    params_diff_parser = params_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(PARAMS_DIFF_HELP, "params/diff"),
+        help=PARAMS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    params_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
+    )
+    params_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help=("New Git commit to compare (defaults to the current workspace)"),
+    )
+    params_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged params as well.",
+    )
+    params_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    params_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    params_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show params path.",
+    )
+    params_diff_parser.set_defaults(func=CmdParamsDiff)

dvc/command/pipeline.py

@@ -1,244 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.exceptions import DvcException
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdPipelineShow(CmdBase):
-    def _show(self, target, commands, outs, locked):
-        import networkx
-        from dvc.stage import Stage
-
-        stage = Stage.load(self.repo, target)
-        G = self.repo.graph
-        stages = networkx.dfs_postorder_nodes(G, stage)
-
-        if locked:
-            stages = [s for s in stages if s.locked]
-
-        for stage in stages:
-            if commands:
-                if stage.cmd is None:
-                    continue
-                logger.info(stage.cmd)
-            elif outs:
-                for out in stage.outs:
-                    logger.info(str(out))
-            else:
-                logger.info(stage.path_in_repo)
-
-    def __build_graph(self, target, commands, outs):
-        import networkx
-        from dvc.stage import Stage
-        from dvc.repo.graph import get_pipeline
-
-        stage = Stage.load(self.repo, target)
-        G = get_pipeline(self.repo.pipelines, stage)
-
-        nodes = []
-        for stage in G:
-            if commands:
-                if stage.cmd is None:
-                    continue
-                nodes.append(stage.cmd)
-            elif outs:
-                for out in stage.outs:
-                    nodes.append(str(out))
-            else:
-                nodes.append(stage.relpath)
-
-        edges = []
-        for from_stage, to_stage in G.edges():
-            if commands:
-                if to_stage.cmd is None:
-                    continue
-                edges.append((from_stage.cmd, to_stage.cmd))
-            elif outs:
-                for from_out in from_stage.outs:
-                    for to_out in to_stage.outs:
-                        edges.append((str(from_out), str(to_out)))
-            else:
-                edges.append((from_stage.relpath, to_stage.relpath))
-
-        return nodes, edges, networkx.is_tree(G)
-
-    def _show_ascii(self, target, commands, outs):
-        from dvc.dagascii import draw
-
-        nodes, edges, _ = self.__build_graph(target, commands, outs)
-
-        if not nodes:
-            return
-
-        draw(nodes, edges)
-
-    def _show_dependencies_tree(self, target, commands, outs):
-        from treelib import Tree
-
-        nodes, edges, is_tree = self.__build_graph(target, commands, outs)
-        if not nodes:
-            return
-        if not is_tree:
-            raise DvcException(
-                "DAG is not a tree, can not print it in tree-structure way, "
-                "please use --ascii instead"
-            )
-
-        tree = Tree()
-        tree.create_node(target, target)  # Root node
-        observe_list = [target]
-        while len(observe_list) > 0:
-            current_root = observe_list[0]
-            for edge in edges:
-                if edge[0] == current_root:
-                    tree.create_node(edge[1], edge[1], parent=current_root)
-                    observe_list.append(edge[1])
-            observe_list.pop(0)
-        tree.show()
-
-    def __write_dot(self, target, commands, outs):
-        import io
-        import networkx
-        from networkx.drawing.nx_pydot import write_dot
-
-        _, edges, _ = self.__build_graph(target, commands, outs)
-        edges = [edge[::-1] for edge in edges]
-
-        simple_g = networkx.DiGraph()
-        simple_g.add_edges_from(edges)
-
-        dot_file = io.StringIO()
-        write_dot(simple_g, dot_file)
-        logger.info(dot_file.getvalue())
-
-    def run(self):
-        if not self.args.targets:
-            self.args.targets = self.default_targets
-
-        for target in self.args.targets:
-            try:
-                if self.args.ascii:
-                    self._show_ascii(
-                        target, self.args.commands, self.args.outs
-                    )
-                elif self.args.dot:
-                    self.__write_dot(
-                        target, self.args.commands, self.args.outs
-                    )
-                elif self.args.tree:
-                    self._show_dependencies_tree(
-                        target, self.args.commands, self.args.outs
-                    )
-                else:
-                    self._show(
-                        target,
-                        self.args.commands,
-                        self.args.outs,
-                        self.args.locked,
-                    )
-            except DvcException:
-                msg = "failed to show pipeline for '{}'".format(target)
-                logger.exception(msg)
-                return 1
-        return 0
-
-
-class CmdPipelineList(CmdBase):
-    def run(self):
-        pipelines = self.repo.pipelines
-        for pipeline in pipelines:
-            for stage in pipeline:
-                logger.info(stage.relpath)
-            if len(pipeline) != 0:
-                logger.info("=" * 80)
-        logger.info("{} pipelines total".format(len(pipelines)))
-
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    PIPELINE_HELP = "Manage pipelines."
-    pipeline_parser = subparsers.add_parser(
-        "pipeline",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_HELP, "pipeline"),
-        help=PIPELINE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    pipeline_subparsers = pipeline_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc pipeline CMD --help` for command-specific help.",
-    )
-
-    fix_subparsers(pipeline_subparsers)
-
-    PIPELINE_SHOW_HELP = "Show pipelines."
-    pipeline_show_parser = pipeline_subparsers.add_parser(
-        "show",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
-        help=PIPELINE_SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
-    pipeline_show_group.add_argument(
-        "-c",
-        "--commands",
-        action="store_true",
-        default=False,
-        help="Print commands instead of paths to DVC-files.",
-    )
-    pipeline_show_group.add_argument(
-        "-o",
-        "--outs",
-        action="store_true",
-        default=False,
-        help="Print output files instead of paths to DVC-files.",
-    )
-    pipeline_show_parser.add_argument(
-        "-l",
-        "--locked",
-        action="store_true",
-        default=False,
-        help="Print locked DVC stages",
-    )
-    pipeline_show_parser.add_argument(
-        "--ascii",
-        action="store_true",
-        default=False,
-        help="Output DAG as ASCII.",
-    )
-    pipeline_show_parser.add_argument(
-        "--dot",
-        action="store_true",
-        default=False,
-        help="Print DAG with .dot format.",
-    )
-    pipeline_show_parser.add_argument(
-        "--tree",
-        action="store_true",
-        default=False,
-        help="Output DAG as Dependencies Tree.",
-    )
-    pipeline_show_parser.add_argument(
-        "targets",
-        nargs="*",
-        help="DVC-files to show pipeline for. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-    )
-    pipeline_show_parser.set_defaults(func=CmdPipelineShow)
-
-    PIPELINE_LIST_HELP = "List pipelines."
-    pipeline_list_parser = pipeline_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
-        help=PIPELINE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/plots.py

@@ -0,0 +1,233 @@
+import argparse
+import logging
+import os
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+from dvc.schema import PLOT_PROPS
+from dvc.utils import format_link
+
+logger = logging.getLogger(__name__)
+
+PAGE_HTML = """<!DOCTYPE html>
+<html>
+<head>
+    <title>DVC Plot</title>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+</head>
+<body>
+    {divs}
+</body>
+</html>"""
+
+DIV_HTML = """<div id = "{id}"></div>
+<script type = "text/javascript">
+    var spec = {vega_json};
+    vegaEmbed('#{id}', spec);
+</script>"""
+
+
+class CmdPlots(CmdBase):
+    def _func(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def _props(self):
+        # Pass only props specified by user, to not shadow ones from plot def
+        props = {p: getattr(self.args, p) for p in PLOT_PROPS}
+        return {k: v for k, v in props.items() if v is not None}
+
+    def run(self):
+        if self.args.show_vega:
+            if not self.args.targets:
+                logger.error("please specify a target for `--show-vega`")
+                return 1
+            if len(self.args.targets) > 1:
+                logger.error(
+                    "you can only specify one target for `--show-vega`"
+                )
+                return 1
+
+        try:
+            plots = self._func(targets=self.args.targets, props=self._props())
+
+            if self.args.show_vega:
+                target = self.args.targets[0]
+                logger.info(plots[target])
+                return 0
+
+            divs = [
+                DIV_HTML.format(id=f"plot{i}", vega_json=plot)
+                for i, plot in enumerate(plots.values())
+            ]
+            html = PAGE_HTML.format(divs="\n".join(divs))
+            path = self.args.out or "plots.html"
+
+            with open(path, "w") as fobj:
+                fobj.write(html)
+
+            logger.info(
+                "file://{}".format(os.path.join(self.repo.root_dir, path))
+            )
+
+        except DvcException:
+            logger.exception("")
+            return 1
+
+        return 0
+
+
+class CmdPlotsShow(CmdPlots):
+    def _func(self, *args, **kwargs):
+        return self.repo.plots.show(*args, **kwargs)
+
+
+class CmdPlotsDiff(CmdPlots):
+    def _func(self, *args, **kwargs):
+        return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
+
+
+class CmdPlotsModify(CmdPlots):
+    def run(self):
+        self.repo.plots.modify(
+            self.args.target, props=self._props(), unset=self.args.unset,
+        )
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PLOTS_HELP = (
+        "Commands to visualize and compare plot metrics in structured files "
+        "(JSON, YAML, CSV, TSV)"
+    )
+
+    plots_parser = subparsers.add_parser(
+        "plots",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_HELP, "plots"),
+        help=PLOTS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_subparsers = plots_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc plots CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(plots_subparsers)
+
+    SHOW_HELP = "Generate plots from metric files."
+    plots_show_parser = plots_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(SHOW_HELP, "plots/show"),
+        help=SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_show_parser.add_argument(
+        "targets",
+        nargs="*",
+        help="Plots files to visualize. Shows all plots by default.",
+    ).complete = completion.FILE
+    _add_props_arguments(plots_show_parser)
+    _add_output_arguments(plots_show_parser)
+    plots_show_parser.set_defaults(func=CmdPlotsShow)
+
+    PLOTS_DIFF_HELP = (
+        "Show multiple versions of plot metrics "
+        "by plotting them in a single image."
+    )
+    plots_diff_parser = plots_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_DIFF_HELP, "plots/diff"),
+        help=PLOTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help="Plots file to visualize. Shows all plots by default.",
+        metavar="<path>",
+    ).complete = completion.FILE
+    plots_diff_parser.add_argument(
+        "revisions", nargs="*", default=None, help="Git commits to plot from",
+    )
+    _add_props_arguments(plots_diff_parser)
+    _add_output_arguments(plots_diff_parser)
+    plots_diff_parser.set_defaults(func=CmdPlotsDiff)
+
+    PLOTS_MODIFY_HELP = "Modify display properties of plot metric files."
+    plots_modify_parser = plots_subparsers.add_parser(
+        "modify",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_MODIFY_HELP, "plots/modify"),
+        help=PLOTS_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_modify_parser.add_argument(
+        "target", help="Metric file to set properties to",
+    ).complete = completion.FILE
+    _add_props_arguments(plots_modify_parser)
+    plots_modify_parser.add_argument(
+        "--unset",
+        nargs="*",
+        metavar="<property>",
+        help="Unset one or more display properties.",
+    )
+    plots_modify_parser.set_defaults(func=CmdPlotsModify)
+
+
+def _add_props_arguments(parser):
+    parser.add_argument(
+        "-t",
+        "--template",
+        nargs="?",
+        default=None,
+        help=(
+            "Special JSON or HTML schema file to inject with the data. "
+            "See {}".format(
+                format_link("https://man.dvc.org/plots#plot-templates")
+            )
+        ),
+        metavar="<path>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-x", default=None, help="Field name for X axis.", metavar="<field>"
+    )
+    parser.add_argument(
+        "-y", default=None, help="Field name for Y axis.", metavar="<field>"
+    )
+    parser.add_argument(
+        "--no-header",
+        action="store_false",
+        dest="header",
+        default=None,  # Use default None to distinguish when it's not used
+        help="Provided CSV or TSV datafile does not have a header.",
+    )
+    parser.add_argument(
+        "--title", default=None, metavar="<text>", help="Plot title."
+    )
+    parser.add_argument(
+        "--x-label", default=None, help="X axis label", metavar="<text>"
+    )
+    parser.add_argument(
+        "--y-label", default=None, help="Y axis label", metavar="<text>"
+    )
+
+
+def _add_output_arguments(parser):
+    parser.add_argument(
+        "-o",
+        "--out",
+        default=None,
+        help="Destination path to save plots to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    parser.add_argument(
+        "--show-vega",
+        action="store_true",
+        default=False,
+        help="Show output in Vega format.",
+    )

dvc/command/remote.py

@@ -1,79 +1,146 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import fix_subparsers
+from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.remote.config import RemoteConfig
+from dvc.config import ConfigError
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
 
-class CmdRemoteConfig(CmdConfig):
+class CmdRemote(CmdConfig):
     def __init__(self, args):
         super().__init__(args)
-        self.remote_config = RemoteConfig(self.config)
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["remote"]:
+            raise ConfigError(f"remote '{self.args.name}' doesn't exists.")
 
 
-class CmdRemoteAdd(CmdRemoteConfig):
+class CmdRemoteAdd(CmdRemote):
     def run(self):
         if self.args.default:
-            logger.info(
-                "Setting '{}' as a default remote.".format(self.args.name)
-            )
-        self.remote_config.add(
-            self.args.name,
-            self.args.url,
-            force=self.args.force,
-            default=self.args.default,
-            level=self.args.level,
-        )
+            logger.info(f"Setting '{self.args.name}' as a default remote.")
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["remote"] and not self.args.force:
+                raise ConfigError(
+                    "remote '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["remote"][self.args.name] = {"url": self.args.url}
+            if self.args.default:
+                conf["core"]["remote"] = self.args.name
+
         return 0
 
 
-class CmdRemoteRemove(CmdRemoteConfig):
+class CmdRemoteRemove(CmdRemote):
     def run(self):
-        self.remote_config.remove(self.args.name, level=self.args.level)
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["remote"][self.args.name]
+
+        # Remove core.remote refs to this remote in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("remote") == self.args.name:
+                    del conf["core"]["remote"]
+
+            if level == self.args.level:
+                break
+
         return 0
 
 
-class CmdRemoteModify(CmdRemoteConfig):
+class CmdRemoteModify(CmdRemote):
     def run(self):
-        self.remote_config.modify(
-            self.args.name,
-            self.args.option,
-            self.args.value,
-            level=self.args.level,
-        )
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            section = conf["remote"][self.args.name]
+            if self.args.unset:
+                section.pop(self.args.option, None)
+            else:
+                section[self.args.option] = self.args.value
         return 0
 
 
-class CmdRemoteDefault(CmdRemoteConfig):
+class CmdRemoteDefault(CmdRemote):
     def run(self):
+
         if self.args.name is None and not self.args.unset:
-            name = self.remote_config.get_default(level=self.args.level)
-            print(name)
+            conf = self.config.load_one(self.args.level)
+            try:
+                print(conf["core"]["remote"])
+            except KeyError:
+                logger.info("No default remote set")
+                return 1
         else:
-            self.remote_config.set_default(
-                self.args.name, unset=self.args.unset, level=self.args.level
-            )
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("remote", None)
+                else:
+                    merged_conf = self.config.load_config_to_level(
+                        self.args.level
+                    )
+                    if (
+                        self.args.name in conf["remote"]
+                        or self.args.name in merged_conf["remote"]
+                    ):
+                        conf["core"]["remote"] = self.args.name
+                    else:
+                        raise ConfigError(
+                            "default remote must be present in remote list."
+                        )
         return 0
 
 
-class CmdRemoteList(CmdRemoteConfig):
+class CmdRemoteList(CmdRemote):
     def run(self):
-        for name, url in self.remote_config.list(
-            level=self.args.level
-        ).items():
-            logger.info("{}\t{}".format(name, url))
+        conf = self.config.load_one(self.args.level)
+        for name, conf in conf["remote"].items():
+            logger.info("{}\t{}".format(name, conf["url"]))
+        return 0
+
+
+class CmdRemoteRename(CmdRemote):
+    def _rename_default(self, conf):
+        if conf["core"].get("remote") == self.args.name:
+            conf["core"]["remote"] = self.args.new
+
+    def run(self):
+        all_config = self.config.load_config_to_level(None)
+        if self.args.new in all_config.get("remote", {}):
+            raise ConfigError(
+                "Rename failed. Remote name '{}' already exists.".format(
+                    {self.args.new}
+                )
+            )
+
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.new] = conf["remote"][self.args.name]
+            del conf["remote"][self.args.name]
+            self._rename_default(conf)
+
+        for level in reversed(self.config.LEVELS):
+            if level == self.args.level:
+                break
+            with self.config.edit(level) as level_conf:
+                self._rename_default(level_conf)
+
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
-    REMOTE_HELP = "Manage remote storage configuration."
+    REMOTE_HELP = "Set up and manage data remotes."
     remote_parser = subparsers.add_parser(
         "remote",
         parents=[parent_parser],
@@ -89,7 +156,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(remote_subparsers)
 
-    REMOTE_ADD_HELP = "Add remote."
+    REMOTE_ADD_HELP = "Add a new data remote."
     remote_add_parser = remote_subparsers.add_parser(
         "add",
         parents=[parent_config_parser, parent_parser],
@@ -120,7 +187,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
-    REMOTE_DEFAULT_HELP = "Set/unset default remote."
+    REMOTE_DEFAULT_HELP = "Set/unset the default data remote."
     remote_default_parser = remote_subparsers.add_parser(
         "default",
         parents=[parent_config_parser, parent_parser],
@@ -140,20 +207,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_default_parser.set_defaults(func=CmdRemoteDefault)
 
-    REMOTE_REMOVE_HELP = "Remove remote."
-    remote_remove_parser = remote_subparsers.add_parser(
-        "remove",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
-        help=REMOTE_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    remote_remove_parser.add_argument(
-        "name", help="Name of the remote to remove."
-    )
-    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
-
-    REMOTE_MODIFY_HELP = "Modify remote."
+    REMOTE_MODIFY_HELP = "Modify the configuration of a data remote."
     remote_modify_parser = remote_subparsers.add_parser(
         "modify",
         parents=[parent_config_parser, parent_parser],
@@ -177,7 +231,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_modify_parser.set_defaults(func=CmdRemoteModify)
 
-    REMOTE_LIST_HELP = "List available remotes."
+    REMOTE_LIST_HELP = "List all available data remotes."
     remote_list_parser = remote_subparsers.add_parser(
         "list",
         parents=[parent_config_parser, parent_parser],
@@ -186,3 +240,27 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_list_parser.set_defaults(func=CmdRemoteList)
+
+    REMOTE_REMOVE_HELP = "Remove a data remote."
+    remote_remove_parser = remote_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
+        help=REMOTE_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_remove_parser.add_argument(
+        "name", help="Name of the remote to remove."
+    )
+    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
+    REMOTE_RENAME_HELP = "Rename a DVC remote"
+    remote_rename_parser = remote_subparsers.add_parser(
+        "rename",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
+        help=REMOTE_RENAME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_rename_parser.add_argument("name", help="Remote to be renamed")
+    remote_rename_parser.add_argument("new", help="New name of the remote")
+    remote_rename_parser.set_defaults(func=CmdRemoteRename)

dvc/command/remove.py

@@ -1,48 +1,28 @@
 import argparse
 import logging
 
-import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
-    def _is_outs_only(self, target):
-        if not self.args.purge:
-            return True
-
-        if self.args.force:
-            return False
-
-        msg = "Are you sure you want to remove '{}' with its outputs?".format(
-            target
-        )
-
-        if prompt.confirm(msg):
-            return False
-
-        raise DvcException(
-            "Cannot purge without a confirmation from the user."
-            " Use `-f` to force."
-        )
-
     def run(self):
         for target in self.args.targets:
             try:
-                outs_only = self._is_outs_only(target)
-                self.repo.remove(target, outs_only=outs_only)
+                self.repo.remove(target, outs=self.args.outs)
             except DvcException:
-                logger.exception("failed to remove '{}'".format(target))
+                logger.exception(f"failed to remove '{target}'")
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove DVC-file outputs."
+    REMOVE_HELP = (
+        "Remove stage entry, remove .gitignore entry and unprotect outputs"
+    )
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -50,29 +30,13 @@ def add_parser(subparsers, parent_parser):
         help=REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remove_parser_group = remove_parser.add_mutually_exclusive_group()
-    remove_parser_group.add_argument(
-        "-o",
+    remove_parser.add_argument(
         "--outs",
         action="store_true",
-        default=True,
-        help="Only remove DVC-file outputs. (Default)",
-    )
-    remove_parser_group.add_argument(
-        "-p",
-        "--purge",
-        action="store_true",
         default=False,
-        help="Remove DVC-file and all its outputs.",
+        help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Force purge.",
-    )
-    remove_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to remove."
-    )
+        "targets", nargs="+", help="DVC-files to remove.",
+    ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -2,13 +2,13 @@ import argparse
 import logging
 import os
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
+from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -35,10 +35,15 @@ class CmdRepro(CmdBase):
                     interactive=self.args.interactive,
                     pipeline=self.args.pipeline,
                     all_pipelines=self.args.all_pipelines,
-                    ignore_build_cache=self.args.ignore_build_cache,
+                    run_cache=not self.args.no_run_cache,
                     no_commit=self.args.no_commit,
                     downstream=self.args.downstream,
                     recursive=self.args.recursive,
+                    force_downstream=self.args.force_downstream,
+                    experiment=self.args.experiment,
+                    queue=self.args.queue,
+                    run_all=self.args.run_all,
+                    jobs=self.args.jobs,
                 )
 
                 if len(stages) == 0:
@@ -57,7 +62,9 @@ class CmdRepro(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Check for changes and reproduce stages and dependencies."
+    REPRO_HELP = (
+        "Reproduce complete or partial pipelines by executing their stages."
+    )
     repro_parser = subparsers.add_parser(
         "repro",
         parents=[parent_parser],
@@ -68,8 +75,8 @@ def add_parser(subparsers, parent_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-file to reproduce. 'Dvcfile' by default.",
-    )
+        help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
+    ).complete = completion.DVC_FILE
     repro_parser.add_argument(
         "-f",
         "--force",
@@ -90,6 +97,7 @@ def add_parser(subparsers, parent_parser):
         "--cwd",
         default=os.path.curdir,
         help="Directory within your repo to reproduce from.",
+        metavar="<path>",
     )
     repro_parser.add_argument(
         "-m",
@@ -135,7 +143,16 @@ def add_parser(subparsers, parent_parser):
         help="Reproduce all stages in the specified directory.",
     )
     repro_parser.add_argument(
-        "--ignore-build-cache",
+        "--no-run-cache",
+        action="store_true",
+        default=False,
+        help=(
+            "Execute stage commands even if they have already been run with "
+            "the same command/dependencies/outputs/etc before."
+        ),
+    )
+    repro_parser.add_argument(
+        "--force-downstream",
         action="store_true",
         default=False,
         help="Reproduce all descendants of a changed stage even if their "
@@ -153,4 +170,23 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Start from the specified stages when reproducing pipelines.",
     )
+    repro_parser.add_argument(
+        "-e",
+        "--experiment",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    repro_parser.add_argument(
+        "--queue", action="store_true", default=False, help=argparse.SUPPRESS
+    )
+    repro_parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    repro_parser.add_argument(
+        "-j", "--jobs", type=int, help=argparse.SUPPRESS, metavar="<number>"
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -1,8 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -17,7 +16,7 @@ class CmdRoot(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    ROOT_HELP = "Relative path to the repository's directory."
+    ROOT_HELP = "Return the relative path to the root of the DVC project."
     root_parser = subparsers.add_parser(
         "root",
         parents=[parent_parser],

dvc/command/run.py

@@ -1,18 +1,14 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdRun(CmdBase):
     def run(self):
-        overwrite = self.args.yes or self.args.overwrite_dvcfile
-
         if not any(
             [
                 self.args.deps,
@@ -20,15 +16,18 @@ class CmdRun(CmdBase):
                 self.args.outs_no_cache,
                 self.args.metrics,
                 self.args.metrics_no_cache,
+                self.args.plots,
+                self.args.plots_no_cache,
                 self.args.outs_persist,
                 self.args.outs_persist_no_cache,
+                self.args.params,
                 self.args.command,
             ]
         ):  # pragma: no cover
             logger.error(
                 "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
-                "`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, "
-                "`command`."
+                "`-m`, `-M`, `-p`, `--plots`, `--plots-no-cache`, "
+                "`--outs-persist`, `--outs-persist-no-cache`, `command`."
             )
             return 1
 
@@ -39,21 +38,25 @@ class CmdRun(CmdBase):
                 outs_no_cache=self.args.outs_no_cache,
                 metrics=self.args.metrics,
                 metrics_no_cache=self.args.metrics_no_cache,
+                plots=self.args.plots,
+                plots_no_cache=self.args.plots_no_cache,
                 deps=self.args.deps,
+                params=self.args.params,
                 fname=self.args.file,
-                cwd=self.args.cwd,
                 wdir=self.args.wdir,
                 no_exec=self.args.no_exec,
-                overwrite=overwrite,
-                ignore_build_cache=self.args.ignore_build_cache,
-                remove_outs=self.args.remove_outs,
+                force=self.args.force,
+                run_cache=not self.args.no_run_cache,
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
                 outs_persist_no_cache=self.args.outs_persist_no_cache,
                 always_changed=self.args.always_changed,
+                name=self.args.name,
+                single_stage=self.args.single_stage,
+                external=self.args.external,
             )
         except DvcException:
-            logger.exception("failed to run command")
+            logger.exception("")
             return 1
 
         return 0
@@ -77,7 +80,7 @@ class CmdRun(CmdBase):
         if " " not in argument or '"' in argument:
             return argument
 
-        return '"{}"'.format(argument)
+        return f'"{argument}"'
 
 
 def add_parser(subparsers, parent_parser):
@@ -95,6 +98,10 @@ def add_parser(subparsers, parent_parser):
         action="append",
         default=[],
         help="Declare dependencies for reproducible cmd.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-n", "--name", help="Stage name.",
     )
     run_parser.add_argument(
         "-o",
@@ -102,6 +109,7 @@ def add_parser(subparsers, parent_parser):
         action="append",
         default=[],
         help="Declare output file or directory.",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "-O",
@@ -110,32 +118,54 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory "
         "(do not put into DVC cache).",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "-p",
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter to use as additional dependency.",
+        metavar="[<filename>:]<params_list>",
     )
     run_parser.add_argument(
         "-m",
         "--metrics",
         action="append",
         default=[],
-        help="Declare output metric file or directory.",
+        help="Declare output metric file.",
+        metavar="<path>",
     )
     run_parser.add_argument(
         "-M",
         "--metrics-no-cache",
         action="append",
         default=[],
-        help="Declare output metric file or directory "
-        "(do not put into DVC cache).",
+        help="Declare output metric file (do not put into DVC cache).",
+        metavar="<path>",
     )
     run_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+        "--plots",
+        action="append",
+        default=[],
+        help="Declare output plot file.",
+        metavar="<path>",
     )
     run_parser.add_argument(
-        "-c", "--cwd", help="Deprecated, use -w and -f instead."
+        "--plots-no-cache",
+        action="append",
+        default=[],
+        help="Declare output plot file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "--file", metavar="<filename>", help=argparse.SUPPRESS,
     )
     run_parser.add_argument(
         "-w",
         "--wdir",
         help="Directory within your repo to run your command in.",
+        metavar="<path>",
     )
     run_parser.add_argument(
         "--no-exec",
@@ -144,30 +174,20 @@ def add_parser(subparsers, parent_parser):
         help="Only create stage file without actually running it.",
     )
     run_parser.add_argument(
-        "-y",
-        "--yes",
+        "-f",
+        "--force",
         action="store_true",
         default=False,
-        help="Deprecated, use --overwrite-dvcfile instead",
+        help="Overwrite existing stage",
     )
     run_parser.add_argument(
-        "--overwrite-dvcfile",
+        "--no-run-cache",
         action="store_true",
         default=False,
-        help="Overwrite existing DVC-file without asking for confirmation.",
-    )
-    run_parser.add_argument(
-        "--ignore-build-cache",
-        action="store_true",
-        default=False,
-        help="Run this stage even if it has been already ran with the same "
-        "command/dependencies/outputs/etc before.",
-    )
-    run_parser.add_argument(
-        "--remove-outs",
-        action="store_true",
-        default=False,
-        help="Deprecated, this is now the default behavior",
+        help=(
+            "Execute the command even if this stage has already been run "
+            "with the same command/dependencies/outputs/etc before."
+        ),
     )
     run_parser.add_argument(
         "--no-commit",
@@ -181,6 +201,7 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory that will not be "
         "removed upon repro.",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "--outs-persist-no-cache",
@@ -188,6 +209,7 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory that will not be "
         "removed upon repro (do not put into DVC cache).",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "--always-changed",
@@ -195,6 +217,18 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Always consider this DVC-file as changed.",
     )
+    run_parser.add_argument(
+        "--single-stage",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    run_parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow outputs that are outside of the DVC repository.",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -2,7 +2,6 @@ import logging
 
 from dvc.command.data_sync import CmdDataBase
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -20,7 +19,7 @@ class CmdDataStatus(CmdDataBase):
         ind = indent * self.STATUS_INDENT
 
         if isinstance(status, str):
-            logger.info("{}{}".format(ind, status))
+            logger.info(f"{ind}{status}")
             return
 
         if isinstance(status, list):
@@ -34,7 +33,7 @@ class CmdDataStatus(CmdDataBase):
             if isinstance(value, str):
                 logger.info("{}{}{}".format(ind, self._normalize(value), key))
             elif value:
-                logger.info("{}{}:".format(ind, key))
+                logger.info(f"{ind}{key}:")
                 self._show(value, indent + 1)
 
     def run(self):
@@ -47,17 +46,24 @@ class CmdDataStatus(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
             )
-            if st:
-                if self.args.quiet:
-                    return 1
-                else:
-                    self._show(st, indent)
+
+            if self.args.quiet:
+                return bool(st)
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(st))
+            elif st:
+                self._show(st, indent)
             else:
                 logger.info(self.UP_TO_DATE_MSG)
 
-        except Exception:
+        except Exception:  # noqa, pylint: disable=broad-except
             logger.exception("failed to obtain data status")
             return 1
         return 0

dvc/command/tag.py

@@ -1,156 +0,0 @@
-import logging
-
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcException
-from dvc.utils import to_yaml_string
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdTagAdd(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                self.repo.tag.add(
-                    self.args.tag,
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-            except DvcException:
-                logger.exception("failed to add tag")
-                return 1
-        return 0
-
-
-class CmdTagRemove(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                self.repo.tag.remove(
-                    self.args.tag,
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-            except DvcException:
-                logger.exception("failed to remove tag")
-                return 1
-        return 0
-
-
-class CmdTagList(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                tags = self.repo.tag.list(
-                    target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-                if tags:
-                    logger.info(to_yaml_string(tags))
-            except DvcException:
-                logger.exception("failed list tags")
-                return 1
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    TAG_HELP = "A set of commands to manage DVC tags."
-    tag_parser = subparsers.add_parser(
-        "tag",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_HELP, "tag"),
-        add_help=False,
-    )
-
-    tag_subparsers = tag_parser.add_subparsers(
-        dest="cmd",
-        help="Use DVC tag CMD --help to display command-specific help.",
-    )
-
-    fix_subparsers(tag_subparsers)
-
-    TAG_ADD_HELP = "Add DVC tag."
-    tag_add_parser = tag_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_ADD_HELP, "tag-add"),
-        help=TAG_ADD_HELP,
-    )
-    tag_add_parser.add_argument("tag", help="Dvc tag.")
-    tag_add_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_add_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Add tag for all dependencies of the specified DVC-file.",
-    )
-    tag_add_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Add tag for subdirectories of the specified directory.",
-    )
-    tag_add_parser.set_defaults(func=CmdTagAdd)
-
-    TAG_REMOVE_HELP = "Remove DVC tag."
-    tag_remove_parser = tag_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
-        help=TAG_REMOVE_HELP,
-    )
-    tag_remove_parser.add_argument("tag", help="Dvc tag.")
-    tag_remove_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_remove_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Remove tag for all dependencies of the specified DVC-file.",
-    )
-    tag_remove_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Remove tag for subdirectories of the specified directory.",
-    )
-    tag_remove_parser.set_defaults(func=CmdTagRemove)
-
-    TAG_LIST_HELP = "List DVC tags."
-    tag_list_parser = tag_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_LIST_HELP, "tag-list"),
-        help=TAG_LIST_HELP,
-    )
-    tag_list_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_list_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="List tags for all dependencies of the specified DVC-file.",
-    )
-    tag_list_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="List tags for subdirectories of the specified directory.",
-    )
-    tag_list_parser.set_defaults(func=CmdTagList)

dvc/command/unprotect.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -15,14 +14,17 @@ class CmdUnprotect(CmdBase):
             try:
                 self.repo.unprotect(target)
             except DvcException:
-                msg = "failed to unprotect '{}'".format(target)
+                msg = f"failed to unprotect '{target}'"
                 logger.exception(msg)
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = "Unprotect data files or directories."
+    UNPROTECT_HELP = (
+        "Unprotect tracked files or directories (when hardlinks or symlinks "
+        "have been enabled with `dvc config cache.type`)"
+    )
     unprotect_parser = subparsers.add_parser(
         "unprotect",
         parents=[parent_parser],
@@ -31,6 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect."
-    )
+        "targets", nargs="+", help="Data files/directories to unprotect.",
+    ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -1,23 +1,25 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdUpdate(CmdBase):
     def run(self):
         ret = 0
-        for target in self.args.targets:
-            try:
-                self.repo.update(target)
-            except DvcException:
-                logger.exception("failed to update '{}'.".format(target))
-                ret = 1
+        try:
+            self.repo.update(
+                targets=self.args.targets,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+            )
+        except DvcException:
+            logger.exception("failed update data")
+            ret = 1
         return ret
 
 
@@ -31,6 +33,19 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to update."
+        "targets", nargs="+", help="DVC-files to update.",
+    ).complete = completion.DVC_FILE
+    update_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    update_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Update all stages in the specified directory.",
     )
     update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -2,21 +2,23 @@ import argparse
 import itertools
 import logging
 import os
+import pathlib
 import platform
 import uuid
-import pathlib
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.scm.base import SCMError
+from dvc.system import System
+from dvc.utils import relpath
+from dvc.utils.pkg import PKG
+from dvc.version import __version__
 
 try:
     import psutil
 except ImportError:
     psutil = None
 
-from dvc.utils import is_binary, relpath
-from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.version import __version__
-from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.system import System
-from dvc.utils.pkg import PKG
 
 logger = logging.getLogger(__name__)
 
@@ -25,12 +27,18 @@ class CmdVersion(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
 
+        package = PKG
+        if PKG is None:
+            package = ""
+        else:
+            package = f"({PKG})"
+
         info = [
-            "DVC version: {}".format(__version__),
-            "Python version: {}".format(platform.python_version()),
-            "Platform: {}".format(platform.platform()),
-            "Binary: {}".format(is_binary()),
-            "Package: {}".format(PKG),
+            f"DVC version: {__version__} {package}",
+            "---------------------------------",
+            f"Platform: Python {platform.python_version()} on "
+            f"{platform.platform()}",
+            f"Supports: {self.get_supported_remotes()}",
         ]
 
         try:
@@ -43,13 +51,13 @@ class CmdVersion(CmdBaseNoRepo):
             # `dvc config cache.shared group`.
             if os.path.exists(repo.cache.local.cache_dir):
                 info.append(
-                    "Cache: {}".format(self.get_linktype_support_info(repo))
+                    "Cache types: {}".format(
+                        self.get_linktype_support_info(repo)
+                    )
                 )
                 if psutil:
                     fs_type = self.get_fs_type(repo.cache.local.cache_dir)
-                    info.append(
-                        "Filesystem type (cache directory): {}".format(fs_type)
-                    )
+                    info.append(f"Cache directory: {fs_type}")
             else:
                 logger.warning(
                     "Unable to detect supported link types, as cache "
@@ -61,10 +69,15 @@ class CmdVersion(CmdBaseNoRepo):
 
         except NotDvcRepoError:
             root_directory = os.getcwd()
+        except SCMError:
+            root_directory = os.getcwd()
+            info.append("Repo: dvc, git (broken)")
+        else:
+            if psutil:
+                fs_root = self.get_fs_type(os.path.abspath(root_directory))
+                info.append(f"Workspace directory: {fs_root}")
 
-        if psutil:
-            fs_root = self.get_fs_type(os.path.abspath(root_directory))
-            info.append("Filesystem type (workspace): {}".format(fs_root))
+            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
 
         logger.info("\n".join(info))
         return 0
@@ -72,7 +85,7 @@ class CmdVersion(CmdBaseNoRepo):
     @staticmethod
     def get_fs_type(path):
         partition = {
-            pathlib.Path(part.mountpoint): (part.fstype, part.device)
+            pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device)
             for part in psutil.disk_partitions(all=True)
         }
 
@@ -107,15 +120,45 @@ class CmdVersion(CmdBaseNoRepo):
                 os.unlink(dst)
             except DvcException:
                 status = "not supported"
-            cache.append("{name} - {status}".format(name=name, status=status))
+
+            if status == "supported":
+                cache.append(name)
         os.remove(src)
 
         return ", ".join(cache)
 
+    @staticmethod
+    def get_supported_remotes():
+        from dvc.tree import TREES
+
+        supported_remotes = []
+        for tree_cls in TREES:
+            if not tree_cls.get_missing_deps():
+                supported_remotes.append(tree_cls.scheme)
+
+        if len(supported_remotes) == len(TREES):
+            return "All remotes"
+
+        if len(supported_remotes) == 1:
+            return supported_remotes
+
+        return ", ".join(supported_remotes)
+
+
+def _get_dvc_repo_info(repo):
+    if repo.config.get("core", {}).get("no_scm", False):
+        return "dvc (no_scm)"
+
+    if repo.root_dir != repo.scm.root_dir:
+        return "dvc (subdir), git"
+
+    return "dvc, git"
+
 
 def add_parser(subparsers, parent_parser):
-    VERSION_HELP = "Show DVC version and system/environment information."
-
+    VERSION_HELP = (
+        "Display the DVC version and system/environment information."
+    )
     version_parser = subparsers.add_parser(
         "version",
         parents=[parent_parser],

dvc/compat.py

@@ -1,42 +0,0 @@
-"""Python version compatibility code"""
-
-# Backport os.fspath() from Python 3.6
-try:
-    from os import fspath  # noqa: F821
-
-    fspath_py35 = lambda s: s  # noqa: E731
-except ImportError:
-
-    def fspath(path):
-        """Return the path representation of a path-like object.
-
-        If str or bytes is passed in, it is returned unchanged. Otherwise the
-        os.PathLike interface is used to get the path representation. If the
-        path representation is not str or bytes, TypeError is raised. If the
-        provided path is not str, bytes, or os.PathLike, TypeError is raised.
-        """
-        if isinstance(path, (str, bytes)):
-            return path
-
-        # Work from the object's type to match method resolution of other magic
-        # methods.
-        path_type = type(path)
-        try:
-            path_repr = path_type.__fspath__(path)
-        except AttributeError:
-            if hasattr(path_type, "__fspath__"):
-                raise
-            else:
-                raise TypeError(
-                    "expected str, bytes or os.PathLike object, "
-                    "not " + path_type.__name__
-                )
-        if isinstance(path_repr, (str, bytes)):
-            return path_repr
-        else:
-            raise TypeError(
-                "expected {}.__fspath__() to return str or bytes, "
-                "not {}".format(path_type.__name__, type(path_repr).__name__)
-            )
-
-    fspath_py35 = fspath

dvc/config.py

@@ -1,17 +1,28 @@
 """DVC config objects."""
-
-import copy
-import errno
 import logging
 import os
 import re
+from contextlib import contextmanager
+from functools import partial
+from urllib.parse import urlparse
 
 import configobj
-from voluptuous import Schema, Required, Optional, Invalid
-from voluptuous import All, Any, Lower, Range, Coerce, Match
+from funcy import cached_property, compact, re_find, walk_values
+from voluptuous import (
+    ALLOW_EXTRA,
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
 
-from dvc.exceptions import DvcException
-from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.path_info import PathInfo
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -20,7 +31,7 @@ class ConfigError(DvcException):
     """DVC config exception."""
 
     def __init__(self, msg):
-        super().__init__("config file error: {}".format(msg))
+        super().__init__(f"config file error: {msg}")
 
 
 class NoRemoteError(ConfigError):
@@ -48,13 +59,12 @@ def supported_cache_type(types):
 
 
 # Checks that value is either true or false and converts it to bool
-Bool = All(
+to_bool = Bool = All(
     Lower,
     Any("true", "false"),
     lambda v: v == "true",
     msg="expected true or false",
 )
-to_bool = Schema(Bool)
 
 
 def Choices(*choices):
@@ -67,7 +77,153 @@ def Choices(*choices):
     return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
 
 
-class Config(object):  # pylint: disable=too-many-instance-attributes
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    Optional("no_traverse"): Bool,  # obsoleted
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+}
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+}
+
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+        Optional("experiments", default=False): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    "access_key_id": str,
+                    "secret_access_key": str,
+                    Optional("listobjects", default=False): Bool,
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "sse_kms_key_id": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_email": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_p12_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+}
+COMPILED_SCHEMA = Schema(SCHEMA)
+
+
+class Config(dict):
     """Class that manages configuration files for a DVC repo.
 
     Args:
@@ -77,176 +233,25 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         validate (bool): optional flag to tell dvc if it should validate the
             config or just load it as is. 'True' by default.
 
-
     Raises:
-        ConfigError: thrown when config has an invalid format.
+        ConfigError: thrown if config has an invalid format.
     """
 
     APPNAME = "dvc"
     APPAUTHOR = "iterative"
 
-    # NOTE: used internally in RemoteLOCAL to know config
-    # location, that url should resolved relative to.
-    PRIVATE_CWD = "_cwd"
+    # In the order they shadow each other
+    LEVELS = ("system", "global", "repo", "local")
 
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
-    CREDENTIALPATH = "credentialpath"
+    def __init__(
+        self, dvc_dir=None, validate=True, tree=None,
+    ):  # pylint: disable=super-init-not-called
+        from dvc.tree.local import LocalTree
 
-    LEVEL_LOCAL = 0
-    LEVEL_REPO = 1
-    LEVEL_GLOBAL = 2
-    LEVEL_SYSTEM = 3
-
-    SECTION_CORE = "core"
-    SECTION_CORE_LOGLEVEL = "loglevel"
-    SECTION_CORE_LOGLEVEL_SCHEMA = All(
-        Lower, Choices("info", "debug", "warning", "error")
-    )
-    SECTION_CORE_REMOTE = "remote"
-    SECTION_CORE_INTERACTIVE = "interactive"
-    SECTION_CORE_ANALYTICS = "analytics"
-    SECTION_CORE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_CORE_HARDLINK_LOCK = "hardlink_lock"
-
-    SECTION_CACHE = "cache"
-    SECTION_CACHE_DIR = "dir"
-    SECTION_CACHE_TYPE = "type"
-    SECTION_CACHE_PROTECTED = "protected"
-    SECTION_CACHE_SHARED = "shared"
-    SECTION_CACHE_SHARED_SCHEMA = All(Lower, Choices("group"))
-    SECTION_CACHE_LOCAL = "local"
-    SECTION_CACHE_S3 = "s3"
-    SECTION_CACHE_GS = "gs"
-    SECTION_CACHE_SSH = "ssh"
-    SECTION_CACHE_HDFS = "hdfs"
-    SECTION_CACHE_AZURE = "azure"
-    SECTION_CACHE_SLOW_LINK_WARNING = "slow_link_warning"
-    SECTION_CACHE_SCHEMA = {
-        SECTION_CACHE_LOCAL: str,
-        SECTION_CACHE_S3: str,
-        SECTION_CACHE_GS: str,
-        SECTION_CACHE_HDFS: str,
-        SECTION_CACHE_SSH: str,
-        SECTION_CACHE_AZURE: str,
-        SECTION_CACHE_DIR: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_CACHE_SHARED: SECTION_CACHE_SHARED_SCHEMA,
-        PRIVATE_CWD: str,
-        Optional(SECTION_CACHE_SLOW_LINK_WARNING, default=True): Bool,
-    }
-
-    SECTION_CORE_SCHEMA = {
-        SECTION_CORE_LOGLEVEL: SECTION_CORE_LOGLEVEL_SCHEMA,
-        SECTION_CORE_REMOTE: Lower,
-        Optional(SECTION_CORE_INTERACTIVE, default=False): Bool,
-        Optional(SECTION_CORE_ANALYTICS, default=True): Bool,
-        SECTION_CORE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        Optional(SECTION_CORE_HARDLINK_LOCK, default=False): Bool,
-    }
-
-    # aws specific options
-    SECTION_AWS_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_AWS_ENDPOINT_URL = "endpointurl"
-    SECTION_AWS_LIST_OBJECTS = "listobjects"
-    SECTION_AWS_REGION = "region"
-    SECTION_AWS_PROFILE = "profile"
-    SECTION_AWS_USE_SSL = "use_ssl"
-    SECTION_AWS_SSE = "sse"
-    SECTION_AWS_ACL = "acl"
-    SECTION_AWS_GRANT_READ = "grant_read"
-    SECTION_AWS_GRANT_READ_ACP = "grant_read_acp"
-    SECTION_AWS_GRANT_WRITE_ACP = "grant_write_acp"
-    SECTION_AWS_GRANT_FULL_CONTROL = "grant_full_control"
-
-    # gcp specific options
-    SECTION_GCP_CREDENTIALPATH = CREDENTIALPATH
-    SECTION_GCP_PROJECTNAME = "projectname"
-
-    # azure specific option
-    SECTION_AZURE_CONNECTION_STRING = "connection_string"
-
-    # Alibabacloud oss options
-    SECTION_OSS_ACCESS_KEY_ID = "oss_key_id"
-    SECTION_OSS_ACCESS_KEY_SECRET = "oss_key_secret"
-    SECTION_OSS_ENDPOINT = "oss_endpoint"
-
-    # GDrive options
-    SECTION_GDRIVE_CLIENT_ID = "gdrive_client_id"
-    SECTION_GDRIVE_CLIENT_SECRET = "gdrive_client_secret"
-    SECTION_GDRIVE_USER_CREDENTIALS_FILE = "gdrive_user_credentials_file"
-
-    SECTION_REMOTE_CHECKSUM_JOBS = "checksum_jobs"
-    SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
-    SECTION_REMOTE_FMT = 'remote "{}"'
-    SECTION_REMOTE_URL = "url"
-    SECTION_REMOTE_USER = "user"
-    SECTION_REMOTE_PORT = "port"
-    SECTION_REMOTE_KEY_FILE = "keyfile"
-    SECTION_REMOTE_TIMEOUT = "timeout"
-    SECTION_REMOTE_PASSWORD = "password"
-    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
-    SECTION_REMOTE_GSS_AUTH = "gss_auth"
-    SECTION_REMOTE_NO_TRAVERSE = "no_traverse"
-    SECTION_REMOTE_VERIFY = "verify"
-    SECTION_REMOTE_SCHEMA = {
-        Required(SECTION_REMOTE_URL): str,
-        SECTION_AWS_REGION: str,
-        SECTION_AWS_PROFILE: str,
-        SECTION_AWS_CREDENTIALPATH: str,
-        SECTION_AWS_ENDPOINT_URL: str,
-        Optional(SECTION_AWS_LIST_OBJECTS, default=False): Bool,
-        Optional(SECTION_AWS_USE_SSL, default=True): Bool,
-        SECTION_AWS_SSE: str,
-        SECTION_AWS_ACL: str,
-        SECTION_AWS_GRANT_READ: str,
-        SECTION_AWS_GRANT_READ_ACP: str,
-        SECTION_AWS_GRANT_WRITE_ACP: str,
-        SECTION_AWS_GRANT_FULL_CONTROL: str,
-        SECTION_GCP_PROJECTNAME: str,
-        SECTION_CACHE_TYPE: supported_cache_type,
-        Optional(SECTION_CACHE_PROTECTED, default=False): Bool,
-        SECTION_REMOTE_CHECKSUM_JOBS: All(Coerce(int), Range(1)),
-        SECTION_REMOTE_USER: str,
-        SECTION_REMOTE_PORT: Coerce(int),
-        SECTION_REMOTE_KEY_FILE: str,
-        SECTION_REMOTE_TIMEOUT: Coerce(int),
-        SECTION_REMOTE_PASSWORD: str,
-        SECTION_REMOTE_ASK_PASSWORD: Bool,
-        SECTION_REMOTE_GSS_AUTH: Bool,
-        SECTION_AZURE_CONNECTION_STRING: str,
-        SECTION_OSS_ACCESS_KEY_ID: str,
-        SECTION_OSS_ACCESS_KEY_SECRET: str,
-        SECTION_OSS_ENDPOINT: str,
-        SECTION_GDRIVE_CLIENT_ID: str,
-        SECTION_GDRIVE_CLIENT_SECRET: str,
-        SECTION_GDRIVE_USER_CREDENTIALS_FILE: str,
-        PRIVATE_CWD: str,
-        SECTION_REMOTE_NO_TRAVERSE: Bool,
-        SECTION_REMOTE_VERIFY: Bool,
-    }
-
-    SECTION_STATE = "state"
-    SECTION_STATE_ROW_LIMIT = "row_limit"
-    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
-    SECTION_STATE_SCHEMA = {
-        SECTION_STATE_ROW_LIMIT: All(Coerce(int), Range(1)),
-        SECTION_STATE_ROW_CLEANUP_QUOTA: All(Coerce(int), Range(0, 100)),
-    }
-
-    SCHEMA = {
-        Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
-        Match(SECTION_REMOTE_REGEX): SECTION_REMOTE_SCHEMA,
-        Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
-        Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-    }
-    COMPILED_SCHEMA = Schema(SCHEMA)
-
-    def __init__(self, dvc_dir=None, validate=True):
         self.dvc_dir = dvc_dir
-        self.validate = validate
 
         if not dvc_dir:
             try:
@@ -258,33 +263,34 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.load()
+        self.wtree = LocalTree(None, {"url": self.dvc_dir})
+        self.tree = tree or self.wtree
 
-    @staticmethod
-    def get_global_config_dir():
-        """Returns global config location. E.g. ~/.config/dvc/config.
+        self.load(validate=validate)
 
-        Returns:
-            str: path to the global config directory.
-        """
-        from appdirs import user_config_dir
+    @classmethod
+    def get_dir(cls, level):
+        from appdirs import user_config_dir, site_config_dir
 
-        return user_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        assert level in ("global", "system")
 
-    @staticmethod
-    def get_system_config_dir():
-        """Returns system config location. E.g. /etc/dvc.conf.
+        if level == "global":
+            return user_config_dir(cls.APPNAME, cls.APPAUTHOR)
+        if level == "system":
+            return site_config_dir(cls.APPNAME, cls.APPAUTHOR)
 
-        Returns:
-            str: path to the system config directory.
-        """
-        from appdirs import site_config_dir
+    @cached_property
+    def files(self):
+        files = {
+            level: os.path.join(self.get_dir(level), self.CONFIG)
+            for level in ("system", "global")
+        }
 
-        return site_config_dir(
-            appname=Config.APPNAME, appauthor=Config.APPAUTHOR
-        )
+        if self.dvc_dir is not None:
+            files["repo"] = os.path.join(self.dvc_dir, self.CONFIG)
+            files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
+
+        return files
 
     @staticmethod
     def init(dvc_dir):
@@ -300,243 +306,164 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def _resolve_cache_path(self, config):
-        cache = config.get(self.SECTION_CACHE)
-        if cache is None:
-            return
-
-        cache_dir = cache.get(self.SECTION_CACHE_DIR)
-        if cache_dir is None:
-            return
-
-        cache[self.PRIVATE_CWD] = os.path.dirname(config.filename)
-
-    def _resolve_paths(self, config):
-        if config.filename is None:
-            return config
-
-        ret = copy.deepcopy(config)
-        self._resolve_cache_path(ret)
-
-        for section in ret.values():
-            if self.SECTION_REMOTE_URL not in section.keys():
-                continue
-
-            section[self.PRIVATE_CWD] = os.path.dirname(ret.filename)
-
-        return ret
-
-    def _load_configs(self):
-        system_config_file = os.path.join(
-            self.get_system_config_dir(), self.CONFIG
-        )
-
-        global_config_file = os.path.join(
-            self.get_global_config_dir(), self.CONFIG
-        )
-
-        self._system_config = configobj.ConfigObj(system_config_file)
-        self._global_config = configobj.ConfigObj(global_config_file)
-        self._repo_config = configobj.ConfigObj()
-        self._local_config = configobj.ConfigObj()
-
-        if not self.dvc_dir:
-            return
-
-        config_file = os.path.join(self.dvc_dir, self.CONFIG)
-        config_local_file = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
-
-        self._repo_config = configobj.ConfigObj(config_file)
-        self._local_config = configobj.ConfigObj(config_local_file)
-
-    @property
-    def config_local_file(self):
-        return self._local_config.filename
-
-    @property
-    def config_file(self):
-        return self._repo_config.filename
-
-    def load(self):
+    def load(self, validate=True):
         """Loads config from all the config files.
 
         Raises:
-            dvc.config.ConfigError: thrown if config has invalid format.
+            ConfigError: thrown if config has an invalid format.
         """
-        self._load_configs()
+        conf = self.load_config_to_level()
 
-        self.config = configobj.ConfigObj()
-        for c in [
-            self._system_config,
-            self._global_config,
-            self._repo_config,
-            self._local_config,
-        ]:
-            c = self._resolve_paths(c)
-            c = self._lower(c)
-            self.config.merge(c)
+        if validate:
+            conf = self.validate(conf)
 
-        if not self.validate:
-            return
+        self.clear()
+        self.update(conf)
 
-        d = self.config.dict()
-        try:
-            d = self.COMPILED_SCHEMA(d)
-        except Invalid as exc:
-            raise ConfigError(str(exc)) from exc
-        self.config = configobj.ConfigObj(d, write_empty_values=True)
+        # Add resolved default cache.dir
+        if not self["cache"].get("dir") and self.dvc_dir:
+            self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
-    def save(self, config=None):
-        """Saves config to config files.
+    def _load_config(self, level):
+        filename = self.files[level]
 
-        Raises:
-            dvc.config.ConfigError: thrown if failed to write config file.
-        """
-        if config is not None:
-            clist = [config]
+        if self.tree.exists(filename, use_dvcignore=False):
+            with self.tree.open(filename) as fobj:
+                conf_obj = configobj.ConfigObj(fobj)
         else:
-            clist = [
-                self._system_config,
-                self._global_config,
-                self._repo_config,
-                self._local_config,
-            ]
+            conf_obj = configobj.ConfigObj()
+        return _parse_remotes(_lower_keys(conf_obj.dict()))
 
-        for conf in clist:
-            self._save(conf)
+    def _save_config(self, level, conf_dict):
+        filename = self.files[level]
 
+        logger.debug(f"Writing '{filename}'.")
+
+        self.tree.makedirs(os.path.dirname(filename))
+
+        config = configobj.ConfigObj(_pack_remotes(conf_dict))
+        with self.tree.open(filename, "wb") as fobj:
+            config.write(fobj)
+        config.filename = filename
+
+    def load_one(self, level):
+        conf = self._load_config(level)
+        conf = self._load_paths(conf, self.files[level])
+
+        # Auto-verify sections
+        for key in COMPILED_SCHEMA.schema:
+            conf.setdefault(key, {})
+
+        return conf
+
+    @staticmethod
+    def _load_paths(conf, filename):
+        abs_conf_dir = os.path.abspath(os.path.dirname(filename))
+
+        def resolve(path):
+            if os.path.isabs(path) or re.match(r"\w+://", path):
+                return path
+            return RelPath(os.path.join(abs_conf_dir, path))
+
+        return Config._map_dirs(conf, resolve)
+
+    @staticmethod
+    def _to_relpath(conf_dir, path):
+        if re.match(r"\w+://", path):
+            return path
+
+        if isinstance(path, RelPath) or not os.path.isabs(path):
+            path = relpath(path, conf_dir)
+
+        return PathInfo(path).as_posix()
+
+    @staticmethod
+    def _save_paths(conf, filename):
+        conf_dir = os.path.dirname(filename)
+        rel = partial(Config._to_relpath, conf_dir)
+
+        return Config._map_dirs(conf, rel)
+
+    @staticmethod
+    def _map_dirs(conf, func):
+        dirs_schema = {
+            "cache": {"dir": func},
+            "remote": {
+                str: {"url": func, "gdrive_user_credentials_file": func}
+            },
+        }
+        return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
+
+    def load_config_to_level(self, level=None):
+        merged_conf = {}
+        for merge_level in self.LEVELS:
+            if merge_level == level:
+                break
+            if merge_level in self.files:
+                _merge(merged_conf, self.load_one(merge_level))
+        return merged_conf
+
+    @contextmanager
+    def edit(self, level="repo"):
+        if level in {"repo", "local"} and self.dvc_dir is None:
+            raise ConfigError("Not inside a DVC repo")
+
+        conf = self.load_one(level)
+        yield conf
+
+        conf = self._save_paths(conf, self.files[level])
+
+        merged_conf = self.load_config_to_level(level)
+        _merge(merged_conf, conf)
+        self.validate(merged_conf)
+
+        self._save_config(level, conf)
         self.load()
 
     @staticmethod
-    def _save(config):
-        if config.filename is None:
-            return
-
-        logger.debug("Writing '{}'.".format(config.filename))
-        dname = os.path.dirname(os.path.abspath(config.filename))
+    def validate(data):
         try:
-            os.makedirs(dname)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                raise
-        config.write()
+            return COMPILED_SCHEMA(data)
+        except Invalid as exc:
+            raise ConfigError(str(exc)) from None
 
-    def unset(self, section, opt=None, level=None, force=False):
-        """Unsets specified option and/or section in the config.
 
-        Args:
-            section (str): section name.
-            opt (str): optional option name.
-            level (int): config level to use.
-            force (bool): don't error-out even if section doesn't exist. False
-                by default.
+def _parse_remotes(conf):
+    result = {"remote": {}}
 
-        Raises:
-            dvc.config.ConfigError: thrown if section doesn't exist and
-                `force != True`.
-        """
-        config = self.get_configobj(level)
-
-        if section not in config.keys():
-            if force:
-                return
-            raise ConfigError("section '{}' doesn't exist".format(section))
-
-        if opt:
-            if opt not in config[section].keys():
-                if force:
-                    return
-                raise ConfigError(
-                    "option '{}.{}' doesn't exist".format(section, opt)
-                )
-            del config[section][opt]
-
-            if not config[section]:
-                del config[section]
+    for section, val in conf.items():
+        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
+        if name:
+            result["remote"][name] = val
         else:
-            del config[section]
+            result[section] = val
 
-        self.save(config)
+    return result
 
-    def set(self, section, opt, value, level=None, force=True):
-        """Sets specified option in the config.
 
-        Args:
-            section (str): section name.
-            opt (str): option name.
-            value: value to set option to.
-            level (int): config level to use.
-            force (bool): set option even if section already exists. True by
-                default.
+def _pack_remotes(conf):
+    # Drop empty sections
+    result = compact(conf)
 
-        Raises:
-            dvc.config.ConfigError: thrown if section already exists and
-                `force != True`.
+    # Transform remote.name -> 'remote "name"'
+    for name, val in conf["remote"].items():
+        result[f'remote "{name}"'] = val
+    result.pop("remote", None)
 
-        """
-        config = self.get_configobj(level)
+    return result
 
-        if section not in config.keys():
-            config[section] = {}
-        elif not force:
-            raise ConfigError(
-                "Section '{}' already exists. Use `-f|--force` to overwrite "
-                "section with new value.".format(section)
-            )
 
-        config[section][opt] = value
-        self.save(config)
+def _merge(into, update):
+    """Merges second dict into first recursively"""
+    for key, val in update.items():
+        if isinstance(into.get(key), dict) and isinstance(val, dict):
+            _merge(into[key], val)
+        else:
+            into[key] = val
 
-    def get(self, section, opt=None, level=None):
-        """Return option value from the config.
 
-        Args:
-            section (str): section name.
-            opt (str): option name.
-            level (int): config level to use.
-
-        Returns:
-            value (str, int): option value.
-        """
-        config = self.get_configobj(level)
-
-        if section not in config.keys():
-            raise ConfigError("section '{}' doesn't exist".format(section))
-
-        if opt not in config[section].keys():
-            raise ConfigError(
-                "option '{}.{}' doesn't exist".format(section, opt)
-            )
-
-        return config[section][opt]
-
-    @staticmethod
-    def _lower(config):
-        new_config = configobj.ConfigObj()
-        for s_key, s_value in config.items():
-            new_s = {}
-            for key, value in s_value.items():
-                new_s[key.lower()] = str(value)
-            new_config[s_key.lower()] = new_s
-        return new_config
-
-    def get_configobj(self, level):
-        configs = {
-            self.LEVEL_LOCAL: self._local_config,
-            self.LEVEL_REPO: self._repo_config,
-            self.LEVEL_GLOBAL: self._global_config,
-            self.LEVEL_SYSTEM: self._system_config,
-        }
-
-        return configs.get(level, self._repo_config)
-
-    def list_options(self, section_regex, option, level=None):
-        ret = {}
-        config = self.get_configobj(level)
-        for section in config.keys():
-            r = re.match(section_regex, section)
-            if r:
-                name = r.group("name")
-                value = config[section].get(option, "")
-                ret[name] = value
-        return ret
+def _lower_keys(data):
+    return {
+        k.lower(): _lower_keys(v) if isinstance(v, dict) else v
+        for k, v in data.items()
+    }

dvc/daemon.py

@@ -3,13 +3,12 @@
 import inspect
 import logging
 import os
+import platform
 import sys
 from subprocess import Popen
 
 from dvc.env import DVC_DAEMON
-from dvc.utils import fix_env
-from dvc.utils import is_binary
-
+from dvc.utils import fix_env, is_binary
 
 logger = logging.getLogger(__name__)
 
@@ -17,6 +16,14 @@ CREATE_NEW_PROCESS_GROUP = 0x00000200
 DETACHED_PROCESS = 0x00000008
 
 
+def _popen(cmd, **kwargs):
+    prefix = [sys.executable]
+    if not is_binary():
+        prefix += [sys.argv[0]]
+
+    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
+
+
 def _spawn_windows(cmd, env):
     from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
 
@@ -25,17 +32,14 @@ def _spawn_windows(cmd, env):
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    Popen(
-        cmd,
-        env=env,
-        close_fds=True,
-        shell=False,
-        creationflags=creationflags,
-        startupinfo=startupinfo,
+    _popen(
+        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
     ).communicate()
 
 
 def _spawn_posix(cmd, env):
+    from dvc.main import main
+
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
@@ -48,7 +52,6 @@ def _spawn_posix(cmd, env):
         os._exit(1)  # pylint: disable=protected-access
 
     os.setsid()
-    os.umask(0)
 
     try:
         pid = os.fork()
@@ -62,13 +65,19 @@ def _spawn_posix(cmd, env):
     sys.stdout.close()
     sys.stderr.close()
 
-    Popen(cmd, env=env, close_fds=True, shell=False).communicate()
+    if platform.system() == "Darwin":
+        # workaround for MacOS bug
+        # https://github.com/iterative/dvc/issues/4294
+        _popen(cmd, env=env).communicate()
+    else:
+        os.environ.update(env)
+        main(cmd)
 
     os._exit(0)  # pylint: disable=protected-access
 
 
 def _spawn(cmd, env):
-    logger.debug("Trying to spawn '{}'".format(cmd))
+    logger.debug(f"Trying to spawn '{cmd}'")
 
     if os.name == "nt":
         _spawn_windows(cmd, env)
@@ -77,7 +86,7 @@ def _spawn(cmd, env):
     else:
         raise NotImplementedError
 
-    logger.debug("Spawned '{}'".format(cmd))
+    logger.debug(f"Spawned '{cmd}'")
 
 
 def daemon(args):
@@ -90,10 +99,7 @@ def daemon(args):
         logger.debug("skipping launching a new daemon.")
         return
 
-    cmd = [sys.executable]
-    if not is_binary():
-        cmd += [sys.argv[0]]
-    cmd += ["daemon", "-q"] + args
+    cmd = ["daemon", "-q"] + args
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])

dvc/dagascii.py

@@ -6,12 +6,9 @@ import os
 import pydoc
 import sys
 
-from grandalf.graphs import Edge
-from grandalf.graphs import Graph
-from grandalf.graphs import Vertex
+from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
-from grandalf.routing import EdgeViewer
-from grandalf.routing import route_with_lines
+from grandalf.routing import EdgeViewer, route_with_lines
 
 from dvc.env import DVC_PAGER
 from dvc.utils import format_link
@@ -40,7 +37,7 @@ def find_pager():
     if env_pager:
         return make_pager(env_pager)
 
-    if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
+    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
         return make_pager(DEFAULT_PAGER_FORMATTED)
 
     logger.warning(
@@ -52,7 +49,7 @@ def find_pager():
     return pydoc.plainpager
 
 
-class VertexViewer(object):
+class VertexViewer:
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.
 
@@ -78,7 +75,7 @@ class VertexViewer(object):
         return self._w
 
 
-class AsciiCanvas(object):
+class AsciiCanvas:
     """Class for drawing in ASCII.
 
     Args:
@@ -95,14 +92,13 @@ class AsciiCanvas(object):
         self.cols = cols
         self.lines = lines
 
-        self.canvas = [[" "] * cols for l in range(lines)]
+        self.canvas = [[" "] * cols for line in range(lines)]
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
-        pager = find_pager()
         lines = map("".join, self.canvas)
         joined_lines = os.linesep.join(lines)
-        pager(joined_lines)
+        return joined_lines
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.
@@ -148,21 +144,21 @@ class AsciiCanvas(object):
                 if dx == 0:
                     y = y0
                 else:
-                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                    y = y0 + int(round((x - x0) * dy / float(dx)))
                 self.point(x, y, char)
         elif y0 < y1:
             for y in range(y0, y1 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                    x = x0 + int(round((y - y0) * dx / float(dy)))
                 self.point(x, y, char)
         else:
             for y in range(y1, y0 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                    x = x1 + int(round((y - y1) * dx / float(dy)))
                 self.point(x, y, char)
 
     def text(self, x, y, text):
@@ -216,7 +212,7 @@ def _build_sugiyama_layout(vertexes, edges):
     # Y
     #
 
-    vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
+    vertexes = {v: Vertex(f" {v} ") for v in vertexes}
     # NOTE: reverting edges to correctly orientate the graph
     edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
     vertexes = vertexes.values()
@@ -226,7 +222,7 @@ def _build_sugiyama_layout(vertexes, edges):
         vertex.view = VertexViewer(vertex.data)
 
     # NOTE: determine min box length to create the best layout
-    minw = min([v.view.w for v in vertexes])
+    minw = min(v.view.w for v in vertexes)
 
     for edge in edges:
         edge.view = EdgeViewer()
@@ -319,4 +315,4 @@ def draw(vertexes, edges):
             int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
         )
 
-    canvas.draw()
+    return canvas.draw()

dvc/data_cloud.py

@@ -2,15 +2,13 @@
 
 import logging
 
-from dvc.config import Config
 from dvc.config import NoRemoteError
-from dvc.remote import Remote
-
+from dvc.remote import get_remote
 
 logger = logging.getLogger(__name__)
 
 
-class DataCloud(object):
+class DataCloud:
     """Class that manages dvc remotes.
 
     Args:
@@ -24,28 +22,14 @@ class DataCloud(object):
     def __init__(self, repo):
         self.repo = repo
 
-    @property
-    def _config(self):
-        return self.repo.config.config
+    def get_remote(self, name=None, command="<command>"):
+        if not name:
+            name = self.repo.config["core"].get("remote")
 
-    @property
-    def _core(self):
-        return self._config.get(Config.SECTION_CORE, {})
+        if name:
+            return self._init_remote(name)
 
-    def get_remote(self, remote=None, command="<command>"):
-        if not remote:
-            remote = self._core.get(Config.SECTION_CORE_REMOTE)
-
-        if remote:
-            return self._init_remote(remote)
-
-        has_non_default_remote = bool(
-            self.repo.config.list_options(
-                Config.SECTION_REMOTE_REGEX, Config.SECTION_REMOTE_URL
-            )
-        )
-
-        if has_non_default_remote:
+        if bool(self.repo.config["remote"]):
             error_msg = (
                 "no remote specified. Setup default remote with\n"
                 "    dvc remote default <remote name>\n"
@@ -60,55 +44,60 @@ class DataCloud(object):
 
         raise NoRemoteError(error_msg)
 
-    def _init_remote(self, remote):
-        return Remote(self.repo, name=remote)
+    def _init_remote(self, name):
+        return get_remote(self.repo, name=name)
 
-    def push(self, cache, jobs=None, remote=None, show_checksums=False):
+    def push(
+        self, cache, jobs=None, remote=None, show_checksums=False,
+    ):
         """Push data items in a cloud-agnostic way.
 
         Args:
             cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to push to.
+            remote (dvc.remote.base.BaseRemote): optional remote to push to.
                 By default remote from core.remote config option is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
+        remote = self.get_remote(remote, "push")
+
         return self.repo.cache.local.push(
-            cache,
-            jobs=jobs,
-            remote=self.get_remote(remote, "push"),
-            show_checksums=show_checksums,
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
         )
 
-    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
+    def pull(
+        self, cache, jobs=None, remote=None, show_checksums=False,
+    ):
         """Pull data items in a cloud-agnostic way.
 
         Args:
             cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
+            remote (dvc.remote.base.BaseRemote): optional remote to pull from.
                 By default remote from core.remote config option is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "pull")
+
         downloaded_items_num = self.repo.cache.local.pull(
             cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
-        if not remote.verify:
+        if not remote.tree.verify:
             self._save_pulled_checksums(cache)
 
         return downloaded_items_num
 
     def _save_pulled_checksums(self, cache):
-        for checksum in cache["local"].keys():
-            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
-            if self.repo.cache.local.exists(cache_file):
-                # We can safely save here, as existing corrupted files will be
-                # removed upon status, while files corrupted during download
-                # will not be moved from tmp_file (see `RemoteBASE.download()`)
+        for checksum in cache.scheme_keys("local"):
+            cache_file = self.repo.cache.local.tree.hash_to_path_info(checksum)
+            if self.repo.cache.local.tree.exists(cache_file):
+                # We can safely save here, as existing corrupted files will
+                # be removed upon status, while files corrupted during
+                # download will not be moved from tmp_file
+                # (see `RemoteBASE.download()`)
                 self.repo.state.save(cache_file, checksum)
 
     def status(self, cache, jobs=None, remote=None, show_checksums=False):
@@ -117,7 +106,7 @@ class DataCloud(object):
         Args:
             cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to compare
+            remote (dvc.remote.base.BaseRemote): optional remote to compare
                 cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in

dvc/dependency/__init__.py

@@ -1,37 +1,42 @@
+from collections import defaultdict
 from urllib.parse import urlparse
 
 import dvc.output as output
-from dvc.dependency.gs import DependencyGS
-from dvc.dependency.hdfs import DependencyHDFS
-from dvc.dependency.http import DependencyHTTP
-from dvc.dependency.https import DependencyHTTPS
-from dvc.dependency.local import DependencyLOCAL
-from dvc.dependency.s3 import DependencyS3
-from dvc.dependency.ssh import DependencySSH
-from dvc.output.base import OutputBase
-from dvc.remote import Remote
+from dvc.dependency.azure import AzureDependency
+from dvc.dependency.gs import GSDependency
+from dvc.dependency.hdfs import HDFSDependency
+from dvc.dependency.http import HTTPDependency
+from dvc.dependency.https import HTTPSDependency
+from dvc.dependency.local import LocalDependency
+from dvc.dependency.param import ParamsDependency
+from dvc.dependency.s3 import S3Dependency
+from dvc.dependency.ssh import SSHDependency
+from dvc.output.base import BaseOutput
 from dvc.scheme import Schemes
-from .repo import DependencyREPO
 
+from ..tree import get_cloud_tree
+from .repo import RepoDependency
 
 DEPS = [
-    DependencyGS,
-    DependencyHDFS,
-    DependencyHTTP,
-    DependencyHTTPS,
-    DependencyS3,
-    DependencySSH,
-    # NOTE: DependencyLOCAL is the default choice
+    AzureDependency,
+    GSDependency,
+    HDFSDependency,
+    HTTPDependency,
+    HTTPSDependency,
+    S3Dependency,
+    SSHDependency,
+    # NOTE: LocalDependency is the default choice
 ]
 
 DEP_MAP = {
-    Schemes.LOCAL: DependencyLOCAL,
-    Schemes.SSH: DependencySSH,
-    Schemes.S3: DependencyS3,
-    Schemes.GS: DependencyGS,
-    Schemes.HDFS: DependencyHDFS,
-    Schemes.HTTP: DependencyHTTP,
-    Schemes.HTTPS: DependencyHTTPS,
+    Schemes.LOCAL: LocalDependency,
+    Schemes.SSH: SSHDependency,
+    Schemes.S3: S3Dependency,
+    Schemes.AZURE: AzureDependency,
+    Schemes.GS: GSDependency,
+    Schemes.HDFS: HDFSDependency,
+    Schemes.HTTP: HTTPDependency,
+    Schemes.HTTPS: HTTPSDependency,
 }
 
 
@@ -40,39 +45,69 @@ DEP_MAP = {
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metric file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
-del SCHEMA[OutputBase.PARAM_CACHE]
-del SCHEMA[OutputBase.PARAM_METRIC]
-SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
+del SCHEMA[BaseOutput.PARAM_CACHE]
+del SCHEMA[BaseOutput.PARAM_METRIC]
+SCHEMA.update(RepoDependency.REPO_SCHEMA)
+SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 
 
 def _get(stage, p, info):
-    parsed = urlparse(p)
+    parsed = urlparse(p) if p else None
+    if parsed and parsed.scheme == "remote":
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
 
-    if parsed.scheme == "remote":
-        remote = Remote(stage.repo, name=parsed.netloc)
-        return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
+    if info and info.get(RepoDependency.PARAM_REPO):
+        repo = info.pop(RepoDependency.PARAM_REPO)
+        return RepoDependency(repo, stage, p, info)
 
-    if info and info.get(DependencyREPO.PARAM_REPO):
-        repo = info.pop(DependencyREPO.PARAM_REPO)
-        return DependencyREPO(repo, stage, p, info)
+    if info and info.get(ParamsDependency.PARAM_PARAMS):
+        params = info.pop(ParamsDependency.PARAM_PARAMS)
+        return ParamsDependency(stage, p, params)
 
     for d in DEPS:
         if d.supported(p):
             return d(stage, p, info)
-    return DependencyLOCAL(stage, p, info)
+    return LocalDependency(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(OutputBase.PARAM_PATH)
+        p = d.pop(BaseOutput.PARAM_PATH, None)
         ret.append(_get(stage, p, d))
     return ret
 
 
 def loads_from(stage, s_list, erepo=None):
-    ret = []
-    for s in s_list:
-        info = {DependencyREPO.PARAM_REPO: erepo} if erepo else {}
-        ret.append(_get(stage, s, info))
-    return ret
+    assert isinstance(s_list, list)
+    info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
+    return [_get(stage, s, info.copy()) for s in s_list]
+
+
+def _merge_params(s_list):
+    d = defaultdict(list)
+    default_file = ParamsDependency.DEFAULT_PARAMS_FILE
+    for key in s_list:
+        if isinstance(key, str):
+            d[default_file].append(key)
+            continue
+        if not isinstance(key, dict):
+            msg = "Only list of str/dict is supported. Got: "
+            msg += f"'{type(key).__name__}'."
+            raise ValueError(msg)
+
+        for k, params in key.items():
+            if not isinstance(params, list):
+                msg = "Expected list of params for custom params file "
+                msg += f"'{k}', got '{type(params).__name__}'."
+                raise ValueError(msg)
+            d[k].extend(params)
+    return d
+
+
+def loads_params(stage, s_list):
+    d = _merge_params(s_list)
+    return [
+        ParamsDependency(stage, path, params) for path, params in d.items()
+    ]

dvc/dependency/azure.py

@@ -0,0 +1,8 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+
+from ..tree.azure import AzureTree
+
+
+class AzureDependency(BaseDependency, BaseOutput):
+    TREE_CLS = AzureTree

dvc/dependency/base.py

@@ -3,29 +3,27 @@ from dvc.exceptions import DvcException
 
 class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = "dependency '{}' does not exist".format(path)
+        msg = f"dependency '{path}' does not exist"
         super().__init__(msg)
 
 
 class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = "dependency '{}' is not a file or directory".format(path)
+        msg = f"dependency '{path}' is not a file or directory"
         super().__init__(msg)
 
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__(
-            "Stage file '{}' cannot be a dependency.".format(path)
-        )
+        super().__init__(f"Stage file '{path}' cannot be a dependency.")
 
 
-class DependencyBase(object):
+class BaseDependency:
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError
     IsNotFileOrDirError = DependencyIsNotFileOrDirError
     IsStageFileError = DependencyIsStageFileError
 
-    def update(self):
+    def update(self, rev=None):
         pass

dvc/dependency/gs.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.gs import OutputGS
+from dvc.dependency.base import BaseDependency
+from dvc.output.gs import GSOutput
 
 
-class DependencyGS(DependencyBase, OutputGS):
+class GSDependency(BaseDependency, GSOutput):
     pass

dvc/dependency/hdfs.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.hdfs import OutputHDFS
+from dvc.dependency.base import BaseDependency
+from dvc.output.hdfs import HDFSOutput
 
 
-class DependencyHDFS(DependencyBase, OutputHDFS):
+class HDFSDependency(BaseDependency, HDFSOutput):
     pass

dvc/dependency/http.py

@@ -1,7 +1,8 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.base import OutputBase
-from dvc.remote.http import RemoteHTTP
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+
+from ..tree.http import HTTPTree
 
 
-class DependencyHTTP(DependencyBase, OutputBase):
-    REMOTE = RemoteHTTP
+class HTTPDependency(BaseDependency, BaseOutput):
+    TREE_CLS = HTTPTree

dvc/dependency/https.py

@@ -1,6 +1,6 @@
-from .http import DependencyHTTP
-from dvc.remote.https import RemoteHTTPS
+from ..tree.https import HTTPSTree
+from .http import HTTPDependency
 
 
-class DependencyHTTPS(DependencyHTTP):
-    REMOTE = RemoteHTTPS
+class HTTPSDependency(HTTPDependency):
+    TREE_CLS = HTTPSTree

dvc/dependency/local.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.local import OutputLOCAL
+from dvc.dependency.base import BaseDependency
+from dvc.output.local import LocalOutput
 
 
-class DependencyLOCAL(DependencyBase, OutputLOCAL):
+class LocalDependency(BaseDependency, LocalOutput):
     pass

dvc/dependency/param.py

@@ -0,0 +1,120 @@
+import os
+from collections import defaultdict
+
+import dpath.util
+import toml
+import yaml
+from voluptuous import Any
+
+from dvc.dependency.local import LocalDependency
+from dvc.exceptions import DvcException
+
+
+class MissingParamsError(DvcException):
+    pass
+
+
+class BadParamFileError(DvcException):
+    pass
+
+
+class ParamsDependency(LocalDependency):
+    PARAM_PARAMS = "params"
+    PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
+    DEFAULT_PARAMS_FILE = "params.yaml"
+    PARAMS_FILE_LOADERS = defaultdict(lambda: yaml.safe_load)
+    PARAMS_FILE_LOADERS.update({".toml": toml.load})
+
+    def __init__(self, stage, path, params):
+        info = {}
+        self.params = []
+        if params:
+            if isinstance(params, list):
+                self.params = params
+            else:
+                assert isinstance(params, dict)
+                self.params = list(params.keys())
+                info = params
+
+        super().__init__(
+            stage,
+            path
+            or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
+            info=info,
+        )
+
+    def fill_values(self, values=None):
+        """Load params values dynamically."""
+        if not values:
+            return
+        for param in self.params:
+            if param in values:
+                self.info[param] = values[param]
+
+    def save(self):
+        super().save()
+        self.info = self.save_info()
+
+    def status(self):
+        status = super().status()
+
+        if status[str(self)] == "deleted":
+            return status
+
+        status = defaultdict(dict)
+        info = self.read_params()
+        for param in self.params:
+            if param not in info.keys():
+                st = "deleted"
+            elif param not in self.info:
+                st = "new"
+            elif info[param] != self.info[param]:
+                st = "modified"
+            else:
+                assert info[param] == self.info[param]
+                continue
+
+            status[str(self)][param] = st
+
+        return status
+
+    def dumpd(self):
+        return {
+            self.PARAM_PATH: self.def_path,
+            self.PARAM_PARAMS: self.info or self.params,
+        }
+
+    def read_params(self):
+        if not self.exists:
+            return {}
+
+        with self.repo.tree.open(self.path_info, "r") as fobj:
+            try:
+                config = self.PARAMS_FILE_LOADERS[
+                    self.path_info.suffix.lower()
+                ](fobj)
+            except (yaml.YAMLError, toml.TomlDecodeError) as exc:
+                raise BadParamFileError(
+                    f"Unable to read parameters from '{self}'"
+                ) from exc
+
+        ret = {}
+        for param in self.params:
+            try:
+                ret[param] = dpath.util.get(config, param, separator=".")
+            except KeyError:
+                pass
+        return ret
+
+    def save_info(self):
+        info = self.read_params()
+
+        missing_params = set(self.params) - set(info.keys())
+        if missing_params:
+            raise MissingParamsError(
+                "Parameters '{}' are missing from '{}'.".format(
+                    ", ".join(missing_params), self,
+                )
+            )
+
+        return info

dvc/dependency/repo.py

@@ -2,28 +2,31 @@ import os
 
 from voluptuous import Required
 
-from .local import DependencyLOCAL
 from dvc.exceptions import OutputNotFoundError
 from dvc.path_info import PathInfo
 
+from .local import LocalDependency
 
-class DependencyREPO(DependencyLOCAL):
+
+class RepoDependency(LocalDependency):
     PARAM_REPO = "repo"
     PARAM_URL = "url"
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
 
     REPO_SCHEMA = {
-        Required(PARAM_URL): str,
-        PARAM_REV: str,
-        PARAM_REV_LOCK: str,
+        PARAM_REPO: {
+            Required(PARAM_URL): str,
+            PARAM_REV: str,
+            PARAM_REV_LOCK: str,
+        }
     }
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, remote, path):
+    def _parse_path(self, tree, path):
         return None
 
     @property
@@ -47,15 +50,24 @@ class DependencyREPO(DependencyLOCAL):
         return external_repo(d["url"], rev=rev)
 
     def _get_checksum(self, locked=True):
+        from dvc.repo.tree import RepoTree
+
         with self._make_repo(locked=locked) as repo:
             try:
                 return repo.find_out_by_relpath(self.def_path).info["md5"]
             except OutputNotFoundError:
                 path = PathInfo(os.path.join(repo.root_dir, self.def_path))
-                # We are polluting our repo cache with some dir listing here
-                return self.repo.cache.local.get_checksum(path)
 
-    def status(self):
+                # we want stream but not fetch, so DVC out directories are
+                # walked, but dir contents is not fetched
+                tree = RepoTree(repo, stream=True)
+
+                # We are polluting our repo cache with some dir listing here
+                if tree.isdir(path):
+                    return self.repo.cache.local.tree.get_hash(path, tree=tree)
+                return tree.get_file_hash(path)
+
+    def workspace_status(self):
         current_checksum = self._get_checksum(locked=True)
         updated_checksum = self._get_checksum(locked=False)
 
@@ -64,6 +76,9 @@ class DependencyREPO(DependencyLOCAL):
 
         return {}
 
+    def status(self):
+        return self.workspace_status()
+
     def save(self):
         pass
 
@@ -73,13 +88,22 @@ class DependencyREPO(DependencyLOCAL):
     def download(self, to):
         with self._make_repo() as repo:
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+                self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
-            if hasattr(repo, "cache"):
-                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
+            cache = self.repo.cache.local
+            with repo.use_cache(cache):
+                _, _, cache_infos = repo.fetch_external([self.def_path])
+            cache.checkout(to.path_info, cache_infos[0])
 
-            repo.pull_to(self.def_path, to.path_info)
+    def update(self, rev=None):
+        if rev:
+            self.def_repo[self.PARAM_REV] = rev
 
-    def update(self):
         with self._make_repo(locked=False) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+            self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
+
+    def changed_checksum(self):
+        # From current repo point of view what describes RepoDependency is its
+        # origin project url and rev_lock, and it makes RepoDependency
+        # immutable, hence its impossible for checksum to change.
+        return False

dvc/dependency/s3.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.s3 import OutputS3
+from dvc.dependency.base import BaseDependency
+from dvc.output.s3 import S3Output
 
 
-class DependencyS3(DependencyBase, OutputS3):
+class S3Dependency(BaseDependency, S3Output):
     pass

dvc/dependency/ssh.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.ssh import OutputSSH
+from dvc.dependency.base import BaseDependency
+from dvc.output.ssh import SSHOutput
 
 
-class DependencySSH(DependencyBase, OutputSSH):
+class SSHDependency(BaseDependency, SSHOutput):
     pass

dvc/dvcfile.py

@@ -0,0 +1,308 @@
+import collections
+import contextlib
+import logging
+import os
+
+from voluptuous import MultipleInvalid
+
+from dvc.exceptions import DvcException
+from dvc.stage import serialize
+from dvc.stage.exceptions import (
+    StageFileBadNameError,
+    StageFileDoesNotExistError,
+    StageFileFormatError,
+    StageFileIsNotDvcFileError,
+)
+from dvc.stage.loader import SingleStageLoader, StageLoader
+from dvc.utils import relpath
+from dvc.utils.collections import apply_diff
+from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update
+
+logger = logging.getLogger(__name__)
+
+DVC_FILE = "Dvcfile"
+DVC_FILE_SUFFIX = ".dvc"
+PIPELINE_FILE = "dvc.yaml"
+PIPELINE_LOCK = "dvc.lock"
+
+
+class LockfileCorruptedError(DvcException):
+    pass
+
+
+def is_valid_filename(path):
+    return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
+        DVC_FILE,
+        PIPELINE_FILE,
+    ]
+
+
+def is_dvc_file(path):
+    return os.path.isfile(path) and (
+        is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
+    )
+
+
+def check_dvc_filename(path):
+    if not is_valid_filename(path):
+        raise StageFileBadNameError(
+            "bad DVC-file name '{}'. DVC-files should be named "
+            "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
+                relpath(path), os.path.basename(path)
+            )
+        )
+
+
+class FileMixin:
+    SCHEMA = None
+
+    def __init__(self, repo, path, **kwargs):
+        self.repo = repo
+        self.path = path
+
+    def __repr__(self):
+        return "{}: {}".format(
+            self.__class__.__name__, relpath(self.path, self.repo.root_dir)
+        )
+
+    def __hash__(self):
+        return hash(self.path)
+
+    def __eq__(self, other):
+        return self.repo == other.repo and os.path.abspath(
+            self.path
+        ) == os.path.abspath(other.path)
+
+    def __str__(self):
+        return f"{self.__class__.__name__}: {self.relpath}"
+
+    @property
+    def relpath(self):
+        return relpath(self.path)
+
+    def exists(self):
+        return self.repo.tree.exists(self.path)
+
+    def _load(self):
+        # it raises the proper exceptions by priority:
+        # 1. when the file doesn't exists
+        # 2. filename is not a DVC-file
+        # 3. path doesn't represent a regular file
+        if not self.exists():
+            raise StageFileDoesNotExistError(self.path)
+        check_dvc_filename(self.path)
+        if not self.repo.tree.isfile(self.path):
+            raise StageFileIsNotDvcFileError(self.path)
+
+        with self.repo.tree.open(self.path) as fd:
+            stage_text = fd.read()
+        d = parse_yaml(stage_text, self.path)
+        self.validate(d, self.relpath)
+        return d, stage_text
+
+    @classmethod
+    def validate(cls, d, fname=None):
+        assert isinstance(cls.SCHEMA, collections.abc.Callable)
+        try:
+            cls.SCHEMA(d)  # pylint: disable=not-callable
+        except MultipleInvalid as exc:
+            raise StageFileFormatError(f"'{fname}' format error: {exc}")
+
+    def remove(self, force=False):  # pylint: disable=unused-argument
+        with contextlib.suppress(FileNotFoundError):
+            os.unlink(self.path)
+
+    def dump(self, stage, **kwargs):
+        raise NotImplementedError
+
+
+class SingleStageFile(FileMixin):
+    from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
+
+    @property
+    def stage(self):
+        data, raw = self._load()
+        return SingleStageLoader.load_stage(self, data, raw)
+
+    @property
+    def stages(self):
+        data, raw = self._load()
+        return SingleStageLoader(self, data, raw)
+
+    def dump(self, stage, **kwargs):
+        """Dumps given stage appropriately in the dvcfile."""
+        from dvc.stage import PipelineStage
+
+        assert not isinstance(stage, PipelineStage)
+        check_dvc_filename(self.path)
+        logger.debug(
+            "Saving information to '{file}'.".format(file=relpath(self.path))
+        )
+        dump_yaml(self.path, serialize.to_single_stage_file(stage))
+        self.repo.scm.track_file(self.relpath)
+
+    def remove_stage(self, stage):  # pylint: disable=unused-argument
+        self.remove()
+
+
+class PipelineFile(FileMixin):
+    """Abstraction for pipelines file, .yaml + .lock combined."""
+
+    from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
+
+    @property
+    def _lockfile(self):
+        return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
+
+    def dump(
+        self, stage, update_pipeline=False, no_lock=False, **kwargs
+    ):  # pylint: disable=arguments-differ
+        """Dumps given stage appropriately in the dvcfile."""
+        from dvc.stage import PipelineStage
+
+        assert isinstance(stage, PipelineStage)
+        check_dvc_filename(self.path)
+
+        if update_pipeline and not stage.is_data_source:
+            self._dump_pipeline_file(stage)
+
+        if not no_lock:
+            self._dump_lockfile(stage)
+
+    def _dump_lockfile(self, stage):
+        self._lockfile.dump(stage)
+
+    def _dump_pipeline_file(self, stage):
+        data = {}
+        if self.exists():
+            with open(self.path) as fd:
+                data = parse_yaml_for_update(fd.read(), self.path)
+        else:
+            logger.info("Creating '%s'", self.relpath)
+            open(self.path, "w+").close()
+
+        data["stages"] = data.get("stages", {})
+        stage_data = serialize.to_pipeline_file(stage)
+        existing_entry = stage.name in data["stages"]
+
+        action = "Modifying" if existing_entry else "Adding"
+        logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
+
+        if existing_entry:
+            orig_stage_data = data["stages"][stage.name]
+            if "meta" in orig_stage_data:
+                stage_data[stage.name]["meta"] = orig_stage_data["meta"]
+            apply_diff(stage_data[stage.name], orig_stage_data)
+        else:
+            data["stages"].update(stage_data)
+
+        dump_yaml(self.path, data)
+        self.repo.scm.track_file(self.relpath)
+
+    @property
+    def stage(self):
+        raise DvcException(
+            "PipelineFile has multiple stages. Please specify it's name."
+        )
+
+    @property
+    def stages(self):
+        data, _ = self._load()
+        lockfile_data = self._lockfile.load()
+        return StageLoader(self, data.get("stages", {}), lockfile_data)
+
+    def remove(self, force=False):
+        if not force:
+            logger.warning("Cannot remove pipeline file.")
+            return
+
+        super().remove()
+        self._lockfile.remove()
+
+    def remove_stage(self, stage):
+        self._lockfile.remove_stage(stage)
+        if not self.exists():
+            return
+
+        with open(self.path, "r") as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+
+        self.validate(d, self.path)
+        if stage.name not in d.get("stages", {}):
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d["stages"][stage.name]
+
+        if d["stages"]:
+            dump_yaml(self.path, d)
+        else:
+            super().remove()
+
+
+class Lockfile(FileMixin):
+    from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
+
+    def load(self):
+        if not self.exists():
+            return {}
+        with self.repo.tree.open(self.path) as fd:
+            data = parse_yaml(fd.read(), self.path)
+        try:
+            self.validate(data, fname=self.relpath)
+        except StageFileFormatError:
+            raise LockfileCorruptedError(
+                f"Lockfile '{self.relpath}' is corrupted."
+            )
+        return data
+
+    def dump(self, stage, **kwargs):
+        stage_data = serialize.to_lockfile(stage)
+        if not self.exists():
+            modified = True
+            logger.info("Generating lock file '%s'", self.relpath)
+            data = stage_data
+            open(self.path, "w+").close()
+        else:
+            with self.repo.tree.open(self.path, "r") as fd:
+                data = parse_yaml_for_update(fd.read(), self.path)
+            modified = data.get(stage.name, {}) != stage_data.get(
+                stage.name, {}
+            )
+            if modified:
+                logger.info("Updating lock file '%s'", self.relpath)
+            data.update(stage_data)
+        dump_yaml(self.path, data)
+        if modified:
+            self.repo.scm.track_file(self.relpath)
+
+    def remove_stage(self, stage):
+        if not self.exists():
+            return
+
+        with open(self.path) as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+        self.validate(d, self.path)
+
+        if stage.name not in d:
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d[stage.name]
+
+        if d:
+            dump_yaml(self.path, d)
+        else:
+            self.remove()
+
+
+class Dvcfile:
+    def __new__(cls, repo, path, **kwargs):
+        assert path
+        assert repo
+
+        _, ext = os.path.splitext(path)
+        if ext in [".yaml", ".yml"]:
+            return PipelineFile(repo, path, **kwargs)
+        # fallback to single stage file for better error messages
+        return SingleStageFile(repo, path, **kwargs)

dvc/exceptions.py

@@ -1,11 +1,20 @@
 """Exceptions raised by the dvc."""
+from funcy import first
 
-from dvc.utils import relpath, format_link
+from dvc.utils import format_link, relpath
 
 
 class DvcException(Exception):
     """Base class for all dvc exceptions."""
 
+    def __init__(self, msg, *args):
+        assert msg
+        super().__init__(msg, *args)
+
+
+class InvalidArgumentError(ValueError, DvcException):
+    """Thrown if arguments are invalid."""
+
 
 class OutputDuplicationError(DvcException):
     """Thrown if a file/directory is specified as an output in more than one
@@ -19,12 +28,17 @@ class OutputDuplicationError(DvcException):
     def __init__(self, output, stages):
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
-        msg = (
-            "file/directory '{}' is specified as an output in more than one "
-            "stage: {}\n"
-            "This is not allowed. Consider using a different output name."
-        ).format(output, "\n    ".join(s.relpath for s in stages))
+        if len(stages) == 1:
+            msg = "output '{}' is already specified in {}.".format(
+                output, first(stages)
+            )
+        else:
+            msg = "output '{}' is already specified in stages:\n{}".format(
+                output, "\n".join(f"\t- {s.addressing}" for s in stages),
+            )
         super().__init__(msg)
+        self.stages = stages
+        self.output = output
 
 
 class OutputNotFoundError(DvcException):
@@ -56,8 +70,8 @@ class StagePathAsOutputError(DvcException):
     def __init__(self, stage, output):
         assert isinstance(output, str)
         super().__init__(
-            "'{stage}' is within an output '{output}' of another stage".format(
-                stage=stage.relpath, output=output
+            "{stage} is within an output '{output}' of another stage".format(
+                stage=stage, output=output
             )
         )
 
@@ -73,10 +87,7 @@ class CircularDependencyError(DvcException):
     def __init__(self, dependency):
         assert isinstance(dependency, str)
 
-        msg = (
-            "file/directory '{}' is specified as an output and as a "
-            "dependency."
-        )
+        msg = "'{}' is specified as an output and as a dependency."
         super().__init__(msg.format(dependency))
 
 
@@ -90,7 +101,7 @@ class ArgumentDuplicationError(DvcException):
 
     def __init__(self, path):
         assert isinstance(path, str)
-        super().__init__("file '{}' is specified more than once.".format(path))
+        super().__init__(f"file '{path}' is specified more than once.")
 
 
 class MoveNotDataSourceError(DvcException):
@@ -112,18 +123,7 @@ class MoveNotDataSourceError(DvcException):
 
 
 class NotDvcRepoError(DvcException):
-    """Thrown if a directory is not a DVC repo.
-
-    Args:
-        root (str): path to the directory.
-    """
-
-    def __init__(self, root):
-        msg = (
-            "you are not inside of a DVC repository "
-            "(checked up to mount point '{}')"
-        )
-        super().__init__(msg.format(root))
+    """Thrown if a directory is not a DVC repo"""
 
 
 class DvcParserError(DvcException):
@@ -136,12 +136,8 @@ class DvcParserError(DvcException):
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
-        stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
-        msg = (
-            "you've introduced a cycle in your pipeline that involves "
-            "the following stages:"
-            "\n"
-            "{stages}".format(stages=stages)
+        msg = "Pipeline has a cycle involving: {}.".format(
+            ", ".join(s.addressing for s in stages)
         )
         super().__init__(msg)
 
@@ -149,7 +145,7 @@ class CyclicGraphError(DvcException):
 class ConfirmRemoveError(DvcException):
     def __init__(self, path):
         super().__init__(
-            "unable to remove '{}' without a confirmation from the user. Use "
+            "unable to remove '{}' without a confirmation. Use "
             "`-f` to force.".format(path)
         )
 
@@ -161,7 +157,7 @@ class InitError(DvcException):
 class ReproductionError(DvcException):
     def __init__(self, dvc_file_name):
         self.path = dvc_file_name
-        super().__init__("failed to reproduce '{}'".format(dvc_file_name))
+        super().__init__(f"failed to reproduce '{dvc_file_name}'")
 
 
 class BadMetricError(DvcException):
@@ -169,7 +165,7 @@ class BadMetricError(DvcException):
         super().__init__(
             "the following metrics do not exist, "
             "are not metric files or are malformed: {paths}".format(
-                paths=", ".join("'{}'".format(path) for path in paths)
+                paths=", ".join(f"'{path}'" for path in paths)
             )
         )
 
@@ -177,17 +173,24 @@ class BadMetricError(DvcException):
 class NoMetricsError(DvcException):
     def __init__(self):
         super().__init__(
-            "no metric files in this repository. "
-            "Use `dvc metrics add` to add a metric file to track."
+            "no metric files in this repository. Use `-m/-M` options for "
+            "`dvc run` to mark stage outputs as  metrics."
         )
 
 
-class StageFileCorruptedError(DvcException):
+class NoPlotsError(DvcException):
+    def __init__(self):
+        super().__init__(
+            "no plots in this repository. Use `--plots/--plots-no-cache` "
+            "options for `dvc run` to mark stage outputs as plots."
+        )
+
+
+class YAMLFileCorruptedError(DvcException):
     def __init__(self, path):
         path = relpath(path)
         super().__init__(
-            "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(path)
+            f"unable to read: '{path}', YAML file structure is corrupted"
         )
 
 
@@ -199,22 +202,15 @@ class RecursiveAddingWhileUsingFilename(DvcException):
 
 
 class OverlappingOutputPathsError(DvcException):
-    def __init__(self, out_1, out_2):
-        super().__init__(
-            "Paths for outs:\n'{}'('{}')\n'{}'('{}')\noverlap. To avoid "
-            "unpredictable behaviour, rerun command with non overlapping outs "
-            "paths.".format(
-                str(out_1),
-                out_1.stage.relpath,
-                str(out_2),
-                out_2.stage.relpath,
-            )
-        )
+    def __init__(self, parent, overlapping_out, message):
+        self.parent = parent
+        self.overlapping_out = overlapping_out
+        super().__init__(message)
 
 
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target):
-        super().__init__("Did you mean `git checkout {}`?".format(target))
+        super().__init__(f"Did you mean `git checkout {target}`?")
 
 
 class ETagMismatchError(DvcException):
@@ -226,10 +222,11 @@ class ETagMismatchError(DvcException):
 
 
 class FileMissingError(DvcException):
-    def __init__(self, path):
+    def __init__(self, path, hint=None):
         self.path = path
+        hint = "" if hint is None else f". {hint}"
         super().__init__(
-            "Can't find '{}' neither locally nor on remote".format(path)
+            f"Can't find '{path}' neither locally nor on remote{hint}"
         )
 
 
@@ -255,25 +252,23 @@ class DownloadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(
-            "{amount} files failed to download".format(amount=amount)
-        )
+        super().__init__(f"{amount} files failed to download")
 
 
 class UploadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(
-            "{amount} files failed to upload".format(amount=amount)
-        )
+        super().__init__(f"{amount} files failed to upload")
 
 
 class CheckoutError(DvcException):
-    def __init__(self, target_infos):
+    def __init__(self, target_infos, stats=None):
+        self.target_infos = target_infos
+        self.stats = stats
         targets = [str(t) for t in target_infos]
         m = (
-            "Checkout failed for following targets:\n {}\nDid you "
+            "Checkout failed for following targets:\n{}\nDid you "
             "forget to fetch?".format("\n".join(targets))
         )
         super().__init__(m)
@@ -286,7 +281,7 @@ class CollectCacheError(DvcException):
 class NoRemoteInExternalRepoError(DvcException):
     def __init__(self, url):
         super().__init__(
-            "No DVC remote is specified in target repository '{}'.".format(url)
+            f"No DVC remote is specified in target repository '{url}'."
         )
 
 
@@ -301,16 +296,23 @@ class NoOutputInExternalRepoError(DvcException):
 
 class HTTPError(DvcException):
     def __init__(self, code, reason):
-        super().__init__("'{} {}'".format(code, reason))
+        super().__init__(f"'{code} {reason}'")
 
 
 class PathMissingError(DvcException):
-    def __init__(self, path, repo):
-        msg = (
-            "The path '{}' does not exist in the target repository '{}'"
-            " neither as an output nor a git-handled file."
-        )
+    default_msg = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " neither as a DVC output nor as a Git-tracked file."
+    )
+    default_msg_dvc_only = (
+        "The path '{}' does not exist in the target repository '{}'"
+        " as an DVC output."
+    )
+
+    def __init__(self, path, repo, dvc_only=False):
+        msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
         super().__init__(msg.format(path, repo))
+        self.dvc_only = dvc_only
 
 
 class RemoteCacheRequiredError(DvcException):
@@ -326,3 +328,19 @@ class RemoteCacheRequiredError(DvcException):
                 format_link("https://man.dvc.org/config#cache"),
             )
         )
+
+
+class IsADirectoryError(DvcException):  # noqa,pylint:disable=redefined-builtin
+    """Raised when a file operation is requested on a directory."""
+
+
+class NoOutputOrStageError(DvcException):
+    """
+    Raised when the target is neither an output nor a stage name in dvc.yaml
+    """
+
+    def __init__(self, target, file):
+        super().__init__(
+            f"'{target}' "
+            f"does not exist as an output or a stage name in '{file}'"
+        )

dvc/external_repo.py

@@ -1,33 +1,45 @@
 import logging
 import os
 import tempfile
+import threading
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
-import threading
+from typing import Iterable
 
-from funcy import retry, suppress, wrap_with, cached_property
+from funcy import cached_property, retry, wrap_with
 
-from dvc.compat import fspath
+from dvc.config import NoRemoteError, NotDvcRepoError
+from dvc.exceptions import (
+    FileMissingError,
+    NoOutputInExternalRepoError,
+    NoRemoteInExternalRepoError,
+    OutputNotFoundError,
+    PathMissingError,
+)
+from dvc.path_info import PathInfo
 from dvc.repo import Repo
-from dvc.config import Config, NoRemoteError, NotDvcRepoError
-from dvc.exceptions import NoRemoteInExternalRepoError
-from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
-from dvc.exceptions import FileMissingError, PathMissingError
-from dvc.remote import RemoteConfig
-from dvc.utils.fs import remove, fs_copy
-from dvc.scm import SCM
-
+from dvc.repo.tree import RepoTree
+from dvc.scm.base import CloneError
+from dvc.scm.git import Git
+from dvc.tree.local import LocalTree
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
 
 @contextmanager
-def external_repo(url, rev=None):
-    path = _cached_clone(url, rev)
+def external_repo(url, rev=None, for_write=False):
+    logger.debug("Creating external repo %s@%s", url, rev)
+    path = _cached_clone(url, rev, for_write=for_write)
+    if not rev:
+        # Local HEAD points to the tip of whatever branch we first cloned from
+        # (which may not be the default branch), use origin/HEAD here to get
+        # the tip of the default branch
+        rev = "refs/remotes/origin/HEAD"
     try:
-        repo = ExternalRepo(path, url)
+        repo = ExternalRepo(path, url, rev, for_write=for_write)
     except NotDvcRepoError:
-        repo = ExternalGitRepo(path, url)
+        repo = ExternalGitRepo(path, url, rev)
 
     try:
         yield repo
@@ -41,79 +53,178 @@ def external_repo(url, rev=None):
         raise PathMissingError(exc.path, url)
     finally:
         repo.close()
+        if for_write:
+            _remove(path)
+
+
+CLONES = {}
+CACHE_DIRS = {}
 
 
 def clean_repos():
     # Outside code should not see cache while we are removing
-    repo_paths = list(REPOS_CACHE.values())
-    REPOS_CACHE.clear()
+    paths = [path for path, _ in CLONES.values()] + list(CACHE_DIRS.values())
+    CLONES.clear()
+    CACHE_DIRS.clear()
 
-    for path in repo_paths:
+    for path in paths:
         _remove(path)
 
 
-class ExternalRepo(Repo):
-    def __init__(self, root_dir, url):
-        super().__init__(root_dir)
-        self.url = url
-        self._set_upstream()
+class BaseExternalRepo:
+    # pylint: disable=no-member
 
-    def pull_to(self, path, to_info):
+    _local_cache = None
+
+    def __str__(self):
+        return self.url
+
+    @property
+    def local_cache(self):
+        if hasattr(self, "cache"):
+            return self.cache.local
+        return self._local_cache
+
+    @contextmanager
+    def use_cache(self, cache):
+        """Use the specified cache in place of default tmpdir cache for
+        download operations.
+        """
+        if hasattr(self, "cache"):
+            save_cache = self.cache.local
+            self.cache.local = cache
+        self._local_cache = cache
+
+        yield
+
+        if hasattr(self, "cache"):
+            self.cache.local = save_cache
+        self._local_cache = None
+
+    @cached_property
+    def repo_tree(self):
+        return RepoTree(self, fetch=True)
+
+    def get_rev(self):
+        if isinstance(self.tree, LocalTree):
+            return self.scm.get_rev()
+        return self.tree.rev
+
+    def fetch_external(self, paths: Iterable, **kwargs):
+        """Fetch specified external repo paths into cache.
+
+        Returns 3-tuple in the form
+            (downloaded, failed, list(cache_infos))
+        where cache_infos can be used as checkout targets for the
+        fetched paths.
+        """
+        download_results = []
+        failed = 0
+
+        paths = [PathInfo(self.root_dir) / path for path in paths]
+
+        def download_update(result):
+            download_results.append(result)
+
+        save_infos = []
+        for path in paths:
+            if not self.repo_tree.exists(path):
+                raise PathMissingError(path, self.url)
+            save_info = self.local_cache.save(
+                path,
+                self.repo_tree,
+                None,
+                save_link=False,
+                download_callback=download_update,
+            )
+            save_infos.append(save_info)
+
+        return sum(download_results), failed, save_infos
+
+    def get_external(self, path, dest):
+        """Convenience wrapper for fetch_external and checkout."""
+        if self.local_cache:
+            # fetch DVC and git files to tmpdir cache, then checkout
+            _, _, save_infos = self.fetch_external([path])
+            self.local_cache.checkout(PathInfo(dest), save_infos[0])
+        else:
+            # git-only erepo with no cache, just copy files directly
+            # to dest
+            path = PathInfo(self.root_dir) / path
+            if not self.repo_tree.exists(path):
+                raise PathMissingError(path, self.url)
+            self.repo_tree.copytree(path, dest)
+
+
+class ExternalRepo(Repo, BaseExternalRepo):
+    def __init__(self, root_dir, url, rev, for_write=False):
+        if for_write:
+            super().__init__(root_dir)
+        else:
+            root_dir = os.path.realpath(root_dir)
+            super().__init__(root_dir, scm=Git(root_dir), rev=rev)
+        self.url = url
+        self._set_cache_dir()
+        self._fix_upstream()
+
+    @wrap_with(threading.Lock())
+    def _set_cache_dir(self):
         try:
-            out = None
-            with suppress(OutputNotFoundError):
-                out = self.find_out_by_relpath(path)
+            cache_dir = CACHE_DIRS[self.url]
+        except KeyError:
+            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
 
-            if out and out.use_cache:
-                self._pull_cached(out, to_info)
-                return
+        self.cache.local.cache_dir = cache_dir
+        self._local_cache = self.cache.local
 
-            # Git handled files can't have absolute path
-            if os.path.isabs(path):
-                raise FileNotFoundError
+    def _fix_upstream(self):
+        if not os.path.isdir(self.url):
+            return
 
-            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
+        try:
+            src_repo = Repo(self.url)
+        except NotDvcRepoError:
+            # If ExternalRepo does not throw NotDvcRepoError and Repo does,
+            # the self.url might be a bare git repo.
+            # NOTE: This will fail to resolve remote with relative path,
+            # same as if it was a remote DVC repo.
+            return
 
-    def _pull_cached(self, out, to_info):
-        with self.state:
-            # Only pull unless all needed cache is present
-            if out.changed_cache():
-                self.cloud.pull(out.get_used_cache())
+        try:
+            remote_name = self.config["core"].get("remote")
+            if remote_name:
+                self._fix_local_remote(src_repo, remote_name)
+            else:
+                self._add_upstream(src_repo)
+        finally:
+            src_repo.close()
 
-            out.path_info = to_info
-            failed = out.checkout()
-            # This might happen when pull haven't really pulled all the files
-            if failed:
-                raise FileNotFoundError
+    def _fix_local_remote(self, src_repo, remote_name):
+        # If a remote URL is relative to the source repo,
+        # it will have changed upon config load and made
+        # relative to this new repo. Restore the old one here.
+        new_remote = self.config["remote"][remote_name]
+        old_remote = src_repo.config["remote"][remote_name]
+        if new_remote["url"] != old_remote["url"]:
+            new_remote["url"] = old_remote["url"]
 
-    def _set_upstream(self):
-        # check if the URL is local and no default remote is present
-        # add default remote pointing to the original repo's cache location
-        if os.path.isdir(self.url):
-            rconfig = RemoteConfig(self.config)
-            if not rconfig.has_default():
-                src_repo = Repo(self.url)
-                try:
-                    rconfig.add(
-                        "auto-generated-upstream",
-                        src_repo.cache.local.cache_dir,
-                        default=True,
-                        level=Config.LEVEL_LOCAL,
-                    )
-                finally:
-                    src_repo.close()
+    def _add_upstream(self, src_repo):
+        # Fill the empty upstream entry with a new remote pointing to the
+        # original repo's cache location.
+        cache_dir = src_repo.cache.local.cache_dir
+        self.config["remote"]["auto-generated-upstream"] = {"url": cache_dir}
+        self.config["core"]["remote"] = "auto-generated-upstream"
 
 
-class ExternalGitRepo:
-    def __init__(self, root_dir, url):
-        self.root_dir = root_dir
+class ExternalGitRepo(BaseExternalRepo):
+    def __init__(self, root_dir, url, rev):
+        self.root_dir = os.path.realpath(root_dir)
         self.url = url
+        self.tree = self.scm.get_tree(rev)
 
     @cached_property
     def scm(self):
-        return SCM(self.root_dir)
+        return Git(self.root_dir)
 
     def close(self):
         if "scm" in self.__dict__:
@@ -122,68 +233,113 @@ class ExternalGitRepo:
     def find_out_by_relpath(self, path):
         raise OutputNotFoundError(path, self)
 
-    def pull_to(self, path, to_info):
-        try:
-            # Git handled files can't have absolute path
-            if os.path.isabs(path):
-                raise FileNotFoundError
-
-            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
-
     @contextmanager
     def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        """Opens a specified resource as a file object."""
+        tree = RepoTree(self)
         try:
-            abs_path = os.path.join(self.root_dir, path)
-            with open(abs_path, mode, encoding=encoding) as fd:
-                yield fd
+            with tree.open(
+                path, mode=mode, encoding=encoding, **kwargs
+            ) as fobj:
+                yield fobj
         except FileNotFoundError:
             raise PathMissingError(path, self.url)
 
 
-REPOS_CACHE = {}
-
-
-@wrap_with(threading.Lock())
-def _cached_clone(url, rev):
+def _cached_clone(url, rev, for_write=False):
     """Clone an external git repo to a temporary directory.
 
     Returns the path to a local temporary directory with the specified
-    revision checked out.
+    revision checked out. If for_write is set prevents reusing this dir via
+    cache.
     """
-    if (url, rev) in REPOS_CACHE:
-        path = REPOS_CACHE[url, rev]
-        _git_pull(path, rev)
-        return path
+    # even if we have already cloned this repo, we may need to
+    # fetch/fast-forward to get specified rev
+    clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
 
-    new_path = tempfile.mkdtemp("dvc-erepo")
+    if not for_write and (url) in CLONES:
+        return CLONES[url][0]
 
-    if url in REPOS_CACHE:
-        # Copy and an existing clean clone
-        # This one unlike shutil.copytree() works with an existing dir
-        copy_tree(REPOS_CACHE[url], new_path)
-        _git_pull(new_path, None)
-    else:
-        # Create a new clone
-        _clone_repo(url, new_path)
-
-        # Save clean clone dir so that we will have access to a default branch
-        clean_clone_path = tempfile.mkdtemp("dvc-erepo")
-        copy_tree(new_path, clean_clone_path)
-        REPOS_CACHE[url] = clean_clone_path
+    # Copy to a new dir to keep the clone clean
+    repo_path = tempfile.mkdtemp("dvc-erepo")
+    logger.debug("erepo: making a copy of %s clone", url)
+    copy_tree(clone_path, repo_path)
 
     # Check out the specified revision
-    if rev is not None:
-        _git_checkout(new_path, rev)
+    if for_write:
+        _git_checkout(repo_path, rev)
+    else:
+        CLONES[url] = (repo_path, shallow)
+    return repo_path
 
-    REPOS_CACHE[url, rev] = new_path
-    return new_path
+
+@wrap_with(threading.Lock())
+def _clone_default_branch(url, rev, for_write=False):
+    """Get or create a clean clone of the url.
+
+    The cloned is reactualized with git pull unless rev is a known sha.
+    """
+    clone_path, shallow = CLONES.get(url, (None, False))
+
+    git = None
+    try:
+        if clone_path:
+            git = Git(clone_path)
+            # Do not pull for known shas, branches and tags might move
+            if not Git.is_sha(rev) or not git.has_rev(rev):
+                if shallow:
+                    # If we are missing a rev in a shallow clone, fallback to
+                    # a full (unshallowed) clone. Since fetching specific rev
+                    # SHAs is only available in certain git versions, if we
+                    # have need to reference multiple specific revs for a
+                    # given repo URL it is easier/safer for us to work with
+                    # full clones in this case.
+                    logger.debug("erepo: unshallowing clone for '%s'", url)
+                    _unshallow(git)
+                    shallow = False
+                    CLONES[url] = (clone_path, shallow)
+                else:
+                    logger.debug("erepo: git pull '%s'", url)
+                    git.pull()
+        else:
+            logger.debug("erepo: git clone '%s' to a temporary dir", url)
+            clone_path = tempfile.mkdtemp("dvc-clone")
+            if not for_write and rev and not Git.is_sha(rev):
+                # If rev is a tag or branch name try shallow clone first
+                try:
+                    git = Git.clone(url, clone_path, shallow_branch=rev)
+                    shallow = True
+                    logger.debug(
+                        "erepo: using shallow clone for branch '%s'", rev
+                    )
+                except CloneError:
+                    pass
+            if not git:
+                git = Git.clone(url, clone_path)
+                shallow = False
+            CLONES[url] = (clone_path, shallow)
+    finally:
+        if git:
+            git.close()
+
+    return clone_path, shallow
+
+
+def _unshallow(git):
+    if git.repo.head.is_detached:
+        # If this is a detached head (i.e. we shallow cloned a tag) switch to
+        # the default branch
+        origin_refs = git.repo.remotes["origin"].refs
+        ref = origin_refs["HEAD"].reference
+        branch_name = ref.name.split("/")[-1]
+        branch = git.repo.create_head(branch_name, ref)
+        branch.set_tracking_branch(ref)
+        branch.checkout()
+    git.pull(unshallow=True)
 
 
 def _git_checkout(repo_path, rev):
-    from dvc.scm import Git
-
+    logger.debug("erepo: git checkout %s@%s", repo_path, rev)
     git = Git(repo_path)
     try:
         git.checkout(rev)
@@ -191,21 +347,6 @@ def _git_checkout(repo_path, rev):
         git.close()
 
 
-def _git_pull(repo_path, rev):
-    import git
-
-    # Do not try to pull in a detached mode
-    if rev and git.Repo.re_hexsha_only.search(rev):
-        return
-
-    git = git.Repo(repo_path)
-    try:
-        msg = git.git.pull()
-        logger.debug("external repo: git pull: {}", msg)
-    finally:
-        git.close()
-
-
 def _remove(path):
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
@@ -213,10 +354,3 @@ def _remove(path):
         os_retry(remove)(path)
     else:
         remove(path)
-
-
-def _clone_repo(url, path):
-    from dvc.scm.git import Git
-
-    git = Git.clone(url, path)
-    git.close()

dvc/ignore.py

@@ -1,17 +1,21 @@
 import logging
 import os
+import re
+from itertools import groupby, takewhile
 
-from funcy import cached_property
-from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
+from pathspec.util import normalize_file
+from pygtrie import StringTrie
 
-from dvc.scm.tree import BaseTree
+from dvc.path_info import PathInfo
+from dvc.pathspec_math import merge_patterns
+from dvc.system import System
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
 
-class DvcIgnore(object):
+class DvcIgnore:
     DVCIGNORE_FILE = ".dvcignore"
 
     def __call__(self, root, dirs, files):
@@ -19,109 +23,225 @@ class DvcIgnore(object):
 
 
 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, ignore_file_path, tree):
+    def __init__(self, pattern_list, dirname):
+
+        self.pattern_list = pattern_list
+        self.dirname = dirname
+        self.prefix = self.dirname + os.sep
+
+        regex_pattern_list = map(
+            GitWildMatchPattern.pattern_to_regex, pattern_list
+        )
+
+        self.ignore_spec = [
+            (ignore, re.compile("|".join(item[0] for item in group)))
+            for ignore, group in groupby(regex_pattern_list, lambda x: x[1])
+            if ignore is not None
+        ]
+
+    @classmethod
+    def from_files(cls, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
-
-        self.ignore_file_path = ignore_file_path
-        self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-
+        dirname = os.path.normpath(os.path.dirname(ignore_file_path))
         with tree.open(ignore_file_path, encoding="utf-8") as fobj:
-            self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
+            path_spec_lines = [
+                line for line in map(str.strip, fobj.readlines()) if line
+            ]
+
+        return cls(path_spec_lines, dirname)
 
     def __call__(self, root, dirs, files):
         files = [f for f in files if not self.matches(root, f)]
-        dirs = [d for d in dirs if not self.matches(root, d)]
+        dirs = [d for d in dirs if not self.matches(root, d, True)]
 
         return dirs, files
 
-    def matches(self, dirname, basename):
-        abs_path = os.path.join(dirname, basename)
-        rel_path = relpath(abs_path, self.dirname)
-
-        if os.pardir + os.sep in rel_path:
+    def matches(self, dirname, basename, is_dir=False):
+        # NOTE: `relpath` is too slow, so we have to assume that both
+        # `dirname` and `self.dirname` are relative or absolute together.
+        if dirname == self.dirname:
+            path = basename
+        elif dirname.startswith(self.prefix):
+            rel = dirname[len(self.prefix) :]
+            # NOTE: `os.path.join` is ~x5.5 slower
+            path = f"{rel}{os.sep}{basename}"
+        else:
             return False
-        return self.ignore_spec.match_file(rel_path)
+
+        if not System.is_unix():
+            path = normalize_file(path)
+        return self.ignore(path, is_dir)
+
+    def ignore(self, path, is_dir):
+        result = False
+        if is_dir:
+            path_dir = f"{path}/"
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path) or pattern.match(path_dir):
+                    result = ignore
+        else:
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path):
+                    result = ignore
+        return result
 
     def __hash__(self):
-        return hash(self.ignore_file_path)
+        return hash(self.dirname + ":" + "\n".join(self.pattern_list))
 
     def __eq__(self, other):
         if not isinstance(other, DvcIgnorePatterns):
             return NotImplemented
+        return (self.dirname == other.dirname) & (
+            self.pattern_list == other.pattern_list
+        )
 
-        return self.ignore_file_path == other.ignore_file_path
+    def __bool__(self):
+        return bool(self.pattern_list)
 
 
-class DvcIgnoreDirs(DvcIgnore):
-    def __init__(self, basenames):
-        self.basenames = set(basenames)
+class DvcIgnoreFilterNoop:
+    def __init__(self, tree, root_dir):
+        pass
 
     def __call__(self, root, dirs, files):
-        dirs = [d for d in dirs if d not in self.basenames]
-
         return dirs, files
 
-    def __hash__(self):
-        return hash(tuple(self.basenames))
+    def is_ignored_dir(self, _):
+        return False
 
-    def __eq__(self, other):
-        if not isinstance(other, DvcIgnoreDirs):
-            return NotImplemented
-
-        return self.basenames == other.basenames
+    def is_ignored_file(self, _):
+        return False
 
 
-class DvcIgnoreFilter(object):
-    def __init__(self, tree):
+class DvcIgnoreFilter:
+    @staticmethod
+    def _is_dvc_repo(root, directory):
+        from dvc.repo import Repo
+
+        return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
+
+    def __init__(self, tree, root_dir):
+        from dvc.repo import Repo
+
+        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+
         self.tree = tree
-        self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
-        for root, dirs, files in self.tree.walk(self.tree.tree_root):
-            self._update(root)
-            dirs[:], files[:] = self(root, dirs, files)
+        self.root_dir = root_dir
+        self.ignores_trie_tree = StringTrie(separator=os.sep)
+        self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
+            default_ignore_patterns, root_dir
+        )
+        self._update(self.root_dir)
 
     def _update(self, dirname):
+        old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
+        matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
+
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if self.tree.exists(ignore_file_path):
-            self.ignores.add(DvcIgnorePatterns(ignore_file_path, self.tree))
+        if not matches and self.tree.exists(
+            ignore_file_path, use_dvcignore=False
+        ):
+            new_pattern = DvcIgnorePatterns.from_files(
+                ignore_file_path, self.tree
+            )
+            if old_pattern:
+                self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
+                    *merge_patterns(
+                        old_pattern.pattern_list,
+                        old_pattern.dirname,
+                        new_pattern.pattern_list,
+                        new_pattern.dirname,
+                    )
+                )
+            else:
+                self.ignores_trie_tree[dirname] = new_pattern
+        elif old_pattern:
+            self.ignores_trie_tree[dirname] = old_pattern
+
+        # NOTE: using `walk` + `break` because tree doesn't have `listdir()`
+        for root, dirs, _ in self.tree.walk(dirname, use_dvcignore=False):
+            self._update_sub_repo(root, dirs)
+            break
+
+    def _update_sub_repo(self, root, dirs):
+        for d in dirs:
+            if self._is_dvc_repo(root, d):
+                old_pattern = self.ignores_trie_tree.longest_prefix(root).value
+                if old_pattern:
+                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
+                        *merge_patterns(
+                            old_pattern.pattern_list,
+                            old_pattern.dirname,
+                            ["/{}/".format(d)],
+                            root,
+                        )
+                    )
+                else:
+                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
+                        ["/{}/".format(d)], root
+                    )
 
     def __call__(self, root, dirs, files):
-        for ignore in self.ignores:
-            dirs, files = ignore(root, dirs, files)
+        ignore_pattern = self._get_trie_pattern(root)
+        if ignore_pattern:
+            return ignore_pattern(root, dirs, files)
+        else:
+            return dirs, files
 
-        return dirs, files
+    def _get_trie_pattern(self, dirname):
+        ignore_pattern = self.ignores_trie_tree.get(dirname)
+        if ignore_pattern:
+            return ignore_pattern
 
+        prefix = self.ignores_trie_tree.longest_prefix(dirname).key
+        if not prefix:
+            # outside of the repo
+            return None
 
-class CleanTree(BaseTree):
-    def __init__(self, tree):
-        self.tree = tree
+        dirs = list(
+            takewhile(
+                lambda path: path != prefix,
+                (parent.fspath for parent in PathInfo(dirname).parents),
+            )
+        )
+        dirs.reverse()
+        dirs.append(dirname)
 
-    @cached_property
-    def dvcignore(self):
-        return DvcIgnoreFilter(self.tree)
+        for parent in dirs:
+            self._update(parent)
 
-    @property
-    def tree_root(self):
-        return self.tree.tree_root
+        return self.ignores_trie_tree.get(dirname)
 
-    def open(self, path, mode="r", encoding="utf-8"):
-        return self.tree.open(path, mode, encoding)
+    def _is_ignored(self, path, is_dir=False):
+        if self._outside_repo(path):
+            return True
+        dirname, basename = os.path.split(os.path.normpath(path))
+        ignore_pattern = self._get_trie_pattern(dirname)
+        if ignore_pattern:
+            return ignore_pattern.matches(dirname, basename, is_dir)
+        else:
+            return False
 
-    def exists(self, path):
-        return self.tree.exists(path)
+    def is_ignored_dir(self, path):
+        path = os.path.abspath(path)
+        if path == self.root_dir:
+            return False
 
-    def isdir(self, path):
-        return self.tree.isdir(path)
+        return self._is_ignored(path, True)
 
-    def isfile(self, path):
-        return self.tree.isfile(path)
+    def is_ignored_file(self, path):
+        return self._is_ignored(path, False)
 
-    def walk(self, top, topdown=True):
-        for root, dirs, files in self.tree.walk(top, topdown):
-            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+    def _outside_repo(self, path):
+        path = PathInfo(path)
 
-            yield root, dirs, files
-
-    def walk_files(self, top):
-        for root, _, files in self.walk(top):
-            for file in files:
-                yield os.path.join(root, file)
+        # paths outside of the repo should be ignored
+        path = relpath(path, self.root_dir)
+        if path.startswith("..") or (
+            os.name == "nt"
+            and not os.path.commonprefix(
+                [os.path.abspath(path), self.root_dir]
+            )
+        ):
+            return True
+        return False

dvc/istextfile.py

@@ -7,13 +7,17 @@
 TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
 
 
-def istextfile(fname, blocksize=512):
+def istextfile(fname, blocksize=512, tree=None):
     """ Uses heuristics to guess whether the given file is text or binary,
         by reading a single block of bytes from the file.
         If more than 30% of the chars in the block are non-text, or there
         are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
-    with open(fname, "rb") as fobj:
+    if tree:
+        open_func = tree.open
+    else:
+        open_func = open
+    with open_func(fname, "rb") as fobj:
         block = fobj.read(blocksize)
 
     if not block:

dvc/lock.py

@@ -5,19 +5,19 @@ import os
 import time
 from datetime import timedelta
 
-import zc.lockfile
 import flufl.lock
+import zc.lockfile
 
 from dvc.exceptions import DvcException
-from dvc.utils import format_link
 from dvc.progress import Tqdm
+from dvc.utils import format_link
 
 DEFAULT_TIMEOUT = 5
 
 FAILED_TO_LOCK_MESSAGE = (
     "cannot perform the command because another DVC process seems to be "
     "running on this project. If that is not the case, manually remove "
-    "`.dvc/lock` and try again."
+    "`.dvc/tmp/lock` and try again."
 )
 
 
@@ -25,7 +25,7 @@ class LockError(DvcException):
     """Thrown when unable to acquire the lock for DVC repo."""
 
 
-class Lock(object):
+class Lock:
     """Class for DVC repo lock.
 
     Uses zc.lockfile as backend.
@@ -47,7 +47,7 @@ class Lock(object):
                 disable=not self._friendly,
                 desc=(
                     "If DVC froze, see `hardlink_lock` in {}".format(
-                        format_link("man.dvc.org/config#core")
+                        format_link("https://man.dvc.org/config#core")
                     )
                 ),
             ):
@@ -88,7 +88,9 @@ class HardlinkLock(flufl.lock.Lock):
         tmp_dir (str): a directory to store claim files.
     """
 
-    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+    def __init__(
+        self, lockfile, tmp_dir=None, **kwargs
+    ):  # pylint: disable=super-init-not-called
         import socket
 
         self._tmp_dir = tmp_dir
@@ -117,7 +119,7 @@ class HardlinkLock(flufl.lock.Lock):
     def lockfile(self):
         return self._lockfile
 
-    def lock(self):
+    def lock(self):  # pylint: disable=arguments-differ
         try:
             super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
         except flufl.lock.TimeOutError:

dvc/logger.py

@@ -1,6 +1,5 @@
 """Manages logging configuration for DVC repo."""
 
-import traceback
 import logging.config
 import logging.handlers
 
@@ -8,7 +7,6 @@ import colorama
 
 from dvc.progress import Tqdm
 
-
 FOOTER = (
     "\n{yellow}Having any troubles?{nc}"
     " Hit us up at {blue}https://dvc.org/support{nc},"
@@ -20,123 +18,109 @@ FOOTER = (
 )
 
 
+def addLoggingLevel(levelName, levelNum, methodName=None):
+    """
+    Adds a new logging level to the `logging` module and the
+    currently configured logging class.
+
+    Based on https://stackoverflow.com/questions/2183233
+    """
+    if methodName is None:
+        methodName = levelName.lower()
+
+    assert not hasattr(logging, levelName)
+    assert not hasattr(logging, methodName)
+    assert not hasattr(logging.getLoggerClass(), methodName)
+
+    def logForLevel(self, message, *args, **kwargs):
+        if self.isEnabledFor(levelNum):
+            # pylint: disable=protected-access
+            self._log(levelNum, message, args, **kwargs)
+
+    def logToRoot(message, *args, **kwargs):
+        logging.log(levelNum, message, *args, **kwargs)
+
+    logging.addLevelName(levelNum, levelName)
+    setattr(logging, levelName, levelNum)
+    setattr(logging.getLoggerClass(), methodName, logForLevel)
+    setattr(logging, methodName, logToRoot)
+
+
 class LoggingException(Exception):
     def __init__(self, record):
         msg = "failed to log {}".format(str(record))
         super().__init__(msg)
 
 
-class ExcludeErrorsFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.WARNING
+def excludeFilter(level):
+    class ExcludeLevelFilter(logging.Filter):
+        def filter(self, record):
+            return record.levelno < level
 
-
-class ExcludeInfoFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.INFO
+    return ExcludeLevelFilter
 
 
 class ColorFormatter(logging.Formatter):
-    """Enable color support when logging to a terminal that supports it.
+    """Spit out colored text in supported terminals.
 
-    Color support on Windows versions that do not support ANSI color codes is
-    enabled by use of the colorama__ library.
+    colorama__ makes ANSI escape character sequences work under Windows.
     See the colorama documentation for details.
 
     __ https://pypi.python.org/pypi/colorama
 
-    For records containing `exc_info`, it will use a custom `_walk_exc` to
-    retrieve the whole traceback.
+    If record has an extra `tb_only` attribute, it will not show the
+    exception cause, just the message and the traceback.
     """
 
     color_code = {
+        "TRACE": colorama.Fore.GREEN,
         "DEBUG": colorama.Fore.BLUE,
-        "INFO": "",
         "WARNING": colorama.Fore.YELLOW,
         "ERROR": colorama.Fore.RED,
         "CRITICAL": colorama.Fore.RED,
     }
 
     def format(self, record):
-        msg = record.msg.format(*record.args) if record.args else record.msg
+        record.message = record.getMessage()
+        msg = self.formatMessage(record)
 
         if record.levelname == "INFO":
             return msg
 
-        if record.levelname == "ERROR" or record.levelname == "CRITICAL":
-            exception, stack_trace = self._parse_exc(record)
+        if record.exc_info:
+            if getattr(record, "tb_only", False):
+                cause = ""
+            else:
+                cause = ": ".join(_iter_causes(record.exc_info[1]))
 
-            return (
-                "{color}{levelname}{nc}: {description}" "{stack_trace}\n"
-            ).format(
-                color=self.color_code.get(record.levelname, ""),
-                nc=colorama.Fore.RESET,
-                levelname=record.levelname,
-                description=self._description(msg, exception),
-                msg=msg,
-                stack_trace=stack_trace,
+            msg = "{message}{separator}{cause}".format(
+                message=msg or "",
+                separator=" - " if msg and cause else "",
+                cause=cause,
             )
 
-        return "{color}{levelname}{nc}: {msg}".format(
-            color=self.color_code.get(record.levelname, ""),
+            if _is_verbose():
+                msg += _stack_trace(record.exc_info)
+
+        return "{asctime}{color}{levelname}{nc}: {msg}".format(
+            asctime=self.formatTime(record, self.datefmt),
+            color=self.color_code[record.levelname],
             nc=colorama.Fore.RESET,
             levelname=record.levelname,
             msg=msg,
         )
 
-    def _current_level(self):
-        return logging.getLogger("dvc").getEffectiveLevel()
+    def formatTime(self, record, datefmt=None):
+        # only show if current level is set to DEBUG
+        # also, skip INFO as it is used for UI
+        if not _is_verbose() or record.levelno == logging.INFO:
+            return ""
 
-    def _is_visible(self, record):
-        return record.levelno >= self._current_level()
-
-    def _description(self, message, exception):
-        description = ""
-
-        if exception and message:
-            description = "{message} - {exception}"
-        elif exception:
-            description = "{exception}"
-        elif message:
-            description = "{message}"
-
-        return description.format(message=message, exception=exception)
-
-    def _walk_exc(self, exc_info):
-        exc = exc_info[1]
-
-        exc_list = [str(exc)]
-
-        while hasattr(exc, "__cause__") and exc.__cause__:
-            exc_list.append(str(exc.__cause__))
-            exc = exc.__cause__
-
-        return exc_list
-
-    def _parse_exc(self, record):
-        tb_only = getattr(record, "tb_only", False)
-
-        if not record.exc_info:
-            return (None, "")
-
-        exc_list = self._walk_exc(record.exc_info)
-        tb = traceback.format_exception(*record.exc_info)
-
-        exception = None if tb_only else ": ".join(exc_list)
-
-        if self._current_level() == logging.DEBUG:
-            stack_trace = (
-                "\n" "{red}{line}{nc}\n" "{stack_trace}" "{red}{line}{nc}"
-            ).format(
-                red=colorama.Fore.RED,
-                nc=colorama.Fore.RESET,
-                line="-" * 60,
-                stack_trace="".join(tb),
-            )
-        else:
-            stack_trace = ""
-
-        return (exception, stack_trace)
+        return "{green}{date}{nc} ".format(
+            green=colorama.Fore.GREEN,
+            date=super().formatTime(record, datefmt),
+            nc=colorama.Fore.RESET,
+        )
 
 
 class LoggerHandler(logging.StreamHandler):
@@ -154,19 +138,59 @@ class LoggerHandler(logging.StreamHandler):
             self.flush()
         except RecursionError:
             raise
-        except Exception:
+        except Exception:  # noqa, pylint: disable=broad-except
             self.handleError(record)
 
 
+def _is_verbose():
+    return (
+        logging.NOTSET
+        < logging.getLogger("dvc").getEffectiveLevel()
+        <= logging.DEBUG
+    )
+
+
+def _iter_causes(exc):
+    while exc:
+        yield str(exc)
+        exc = exc.__cause__
+
+
+def _stack_trace(exc_info):
+    import traceback
+
+    return (
+        "\n"
+        "{red}{line}{nc}\n"
+        "{trace}"
+        "{red}{line}{nc}".format(
+            red=colorama.Fore.RED,
+            line="-" * 60,
+            trace="".join(traceback.format_exception(*exc_info)),
+            nc=colorama.Fore.RESET,
+        )
+    )
+
+
+def disable_other_loggers():
+    logging.captureWarnings(True)
+    root = logging.root
+    for (logger_name, logger) in root.manager.loggerDict.items():
+        if logger_name != "dvc" and not logger_name.startswith("dvc."):
+            logger.disabled = True
+
+
 def setup(level=logging.INFO):
     colorama.init()
 
+    addLoggingLevel("TRACE", logging.DEBUG - 5)
     logging.config.dictConfig(
         {
             "version": 1,
             "filters": {
-                "exclude_errors": {"()": ExcludeErrorsFilter},
-                "exclude_info": {"()": ExcludeInfoFilter},
+                "exclude_errors": {"()": excludeFilter(logging.WARNING)},
+                "exclude_info": {"()": excludeFilter(logging.INFO)},
+                "exclude_debug": {"()": excludeFilter(logging.DEBUG)},
             },
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
@@ -184,6 +208,13 @@ def setup(level=logging.INFO):
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_info"],
                 },
+                "console_trace": {
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "TRACE",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_debug"],
+                },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
                     "level": "WARNING",
@@ -197,25 +228,11 @@ def setup(level=logging.INFO):
                     "handlers": [
                         "console_info",
                         "console_debug",
-                        "console_errors",
-                    ],
-                },
-                "paramiko": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
-                        "console_errors",
-                    ],
-                },
-                "flufl.lock": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
+                        "console_trace",
                         "console_errors",
                     ],
                 },
             },
+            "disable_existing_loggers": False,
         }
     )

dvc/main.py

@@ -6,10 +6,10 @@ import logging
 from dvc import analytics
 from dvc.cli import parse_args
 from dvc.config import ConfigError
-from dvc.exceptions import DvcParserError
-from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
-from dvc.logger import FOOTER
+from dvc.logger import FOOTER, disable_other_loggers
+from dvc.tree.pool import close_pools
 from dvc.utils import format_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
@@ -18,11 +18,10 @@ from dvc.utils import format_link
 
 "".encode("idna")
 
-
 logger = logging.getLogger("dvc")
 
 
-def main(argv=None):
+def main(argv=None):  # noqa: C901
     """Run dvc CLI command.
 
     Args:
@@ -32,17 +31,24 @@ def main(argv=None):
         int: command's return code.
     """
     args = None
-    cmd = None
+    disable_other_loggers()
 
     outerLogLevel = logger.level
     try:
         args = parse_args(argv)
 
+        level = None
         if args.quiet:
-            logger.setLevel(logging.CRITICAL)
+            level = logging.CRITICAL
+        elif args.verbose == 1:
+            level = logging.DEBUG
+        elif args.verbose > 1:
+            level = logging.TRACE
 
-        elif args.verbose:
-            logger.setLevel(logging.DEBUG)
+        if level is not None:
+            logger.setLevel(level)
+
+        logger.trace(args)
 
         cmd = args.func(args)
         ret = cmd.run()
@@ -57,8 +63,11 @@ def main(argv=None):
         ret = 253
     except DvcParserError:
         ret = 254
-    except Exception as exc:  # pylint: disable=broad-except
-        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+    except DvcException:
+        ret = 255
+        logger.exception("")
+    except OSError as exc:
+        if exc.errno == errno.EMFILE:
             logger.exception(
                 "too many open files, please visit "
                 "{} to see how to handle this "
@@ -70,17 +79,26 @@ def main(argv=None):
         else:
             logger.exception("unexpected error")
         ret = 255
+    except Exception:  # noqa, pylint: disable=broad-except
+        logger.exception("unexpected error")
+        ret = 255
+
+    try:
+        if ret != 0:
+            logger.info(FOOTER)
+
+        if analytics.is_enabled():
+            analytics.collect_and_send_report(args, ret)
+
+        return ret
     finally:
         logger.setLevel(outerLogLevel)
 
+        # Closing pools by-hand to prevent weird messages when closing SSH
+        # connections. See https://github.com/iterative/dvc/issues/3248 for
+        # more info.
+        close_pools()
+
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()
-
-    if ret != 0:
-        logger.info(FOOTER)
-
-    if analytics.is_enabled():
-        analytics.collect_and_send_report(args, ret)
-
-    return ret

dvc/output/__init__.py

@@ -1,34 +1,44 @@
+from collections import defaultdict
 from urllib.parse import urlparse
-from voluptuous import Any, Required
 
-from dvc.output.base import OutputBase
-from dvc.output.gs import OutputGS
-from dvc.output.hdfs import OutputHDFS
-from dvc.output.local import OutputLOCAL
-from dvc.output.s3 import OutputS3
-from dvc.output.ssh import OutputSSH
-from dvc.remote import Remote
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
+from funcy import collecting, project
+from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
+
+from dvc.output.base import BaseOutput
+from dvc.output.gs import GSOutput
+from dvc.output.hdfs import HDFSOutput
+from dvc.output.local import LocalOutput
+from dvc.output.s3 import S3Output
+from dvc.output.ssh import SSHOutput
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
+from ..tree.hdfs import HDFSTree
+from ..tree.local import LocalTree
+from ..tree.s3 import S3Tree
+
 OUTS = [
-    OutputHDFS,
-    OutputS3,
-    OutputGS,
-    OutputSSH,
-    # NOTE: OutputLOCAL is the default choice
+    HDFSOutput,
+    S3Output,
+    GSOutput,
+    SSHOutput,
+    # NOTE: LocalOutput is the default choice
 ]
 
 OUTS_MAP = {
-    Schemes.HDFS: OutputHDFS,
-    Schemes.S3: OutputS3,
-    Schemes.GS: OutputGS,
-    Schemes.SSH: OutputSSH,
-    Schemes.LOCAL: OutputLOCAL,
+    Schemes.HDFS: HDFSOutput,
+    Schemes.S3: S3Output,
+    Schemes.GS: GSOutput,
+    Schemes.SSH: SSHOutput,
+    Schemes.LOCAL: LocalOutput,
 }
 
+CHECKSUM_SCHEMA = Any(
+    None,
+    And(str, Length(max=0), SetTo(None)),
+    And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
+)
+
 # NOTE: currently there are only 3 possible checksum names:
 #
 #    1) md5 (LOCAL, SSH, GS);
@@ -37,36 +47,36 @@ OUTS_MAP = {
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
-CHECKSUM_SCHEMA = {
-    RemoteLOCAL.PARAM_CHECKSUM: Any(str, None),
-    RemoteS3.PARAM_CHECKSUM: Any(str, None),
-    RemoteHDFS.PARAM_CHECKSUM: Any(str, None),
+CHECKSUMS_SCHEMA = {
+    LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }
 
-TAGS_SCHEMA = {str: CHECKSUM_SCHEMA}
-
-SCHEMA = CHECKSUM_SCHEMA.copy()
-SCHEMA[Required(OutputBase.PARAM_PATH)] = str
-SCHEMA[OutputBase.PARAM_CACHE] = bool
-SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
-SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA
-SCHEMA[OutputBase.PARAM_PERSIST] = bool
+SCHEMA = CHECKSUMS_SCHEMA.copy()
+SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
+SCHEMA[BaseOutput.PARAM_CACHE] = bool
+SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
+SCHEMA[BaseOutput.PARAM_PLOT] = bool
+SCHEMA[BaseOutput.PARAM_PERSIST] = bool
 
 
-def _get(stage, p, info, cache, metric, persist=False, tags=None):
+def _get(
+    stage, p, info=None, cache=True, metric=False, plot=False, persist=False
+):
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        remote = Remote(stage.repo, name=parsed.netloc)
-        return OUTS_MAP[remote.scheme](
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return OUTS_MAP[tree.scheme](
             stage,
             p,
             info,
             cache=cache,
-            remote=remote,
+            tree=tree,
             metric=metric,
+            plot=plot,
             persist=persist,
-            tags=tags,
         )
 
     for o in OUTS:
@@ -76,31 +86,31 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
                 p,
                 info,
                 cache=cache,
-                remote=None,
+                tree=None,
                 metric=metric,
+                plot=plot,
                 persist=persist,
-                tags=tags,
             )
-    return OutputLOCAL(
+    return LocalOutput(
         stage,
         p,
         info,
         cache=cache,
-        remote=None,
+        tree=None,
         metric=metric,
+        plot=plot,
         persist=persist,
-        tags=tags,
     )
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(OutputBase.PARAM_PATH)
-        cache = d.pop(OutputBase.PARAM_CACHE, True)
-        metric = d.pop(OutputBase.PARAM_METRIC, False)
-        persist = d.pop(OutputBase.PARAM_PERSIST, False)
-        tags = d.pop(OutputBase.PARAM_TAGS, None)
+        p = d.pop(BaseOutput.PARAM_PATH)
+        cache = d.pop(BaseOutput.PARAM_CACHE, True)
+        metric = d.pop(BaseOutput.PARAM_METRIC, False)
+        plot = d.pop(BaseOutput.PARAM_PLOT, False)
+        persist = d.pop(BaseOutput.PARAM_PERSIST, False)
         ret.append(
             _get(
                 stage,
@@ -108,24 +118,67 @@ def loadd_from(stage, d_list):
                 info=d,
                 cache=cache,
                 metric=metric,
+                plot=plot,
                 persist=persist,
-                tags=tags,
             )
         )
     return ret
 
 
-def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
-    ret = []
-    for s in s_list:
-        ret.append(
-            _get(
-                stage,
-                s,
-                info={},
-                cache=use_cache,
-                metric=metric,
-                persist=persist,
-            )
+def loads_from(
+    stage, s_list, use_cache=True, metric=False, plot=False, persist=False
+):
+    return [
+        _get(
+            stage,
+            s,
+            info={},
+            cache=use_cache,
+            metric=metric,
+            plot=plot,
+            persist=persist,
         )
-    return ret
+        for s in s_list
+    ]
+
+
+def _split_dict(d, keys):
+    return project(d, keys), project(d, d.keys() - keys)
+
+
+def _merge_data(s_list):
+    d = defaultdict(dict)
+    for key in s_list:
+        if isinstance(key, str):
+            d[key].update({})
+            continue
+        if not isinstance(key, dict):
+            raise ValueError(f"'{type(key).__name__}' not supported.")
+
+        for k, flags in key.items():
+            if not isinstance(flags, dict):
+                raise ValueError(
+                    f"Expected dict for '{k}', got: '{type(flags).__name__}'"
+                )
+            d[k].update(flags)
+    return d
+
+
+@collecting
+def load_from_pipeline(stage, s_list, typ="outs"):
+    if typ not in (stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS):
+        raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
+
+    metric = typ == stage.PARAM_METRICS
+    plot = typ == stage.PARAM_PLOTS
+
+    d = _merge_data(s_list)
+
+    for path, flags in d.items():
+        plt_d = {}
+        if plot:
+            from dvc.schema import PLOT_PROPS
+
+            plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
+        extra = project(flags, ["cache", "persist"])
+        yield _get(stage, path, {}, plot=plt_d or plot, metric=metric, **extra)

dvc/output/base.py

@@ -1,52 +1,68 @@
 import logging
-from urllib.parse import urlparse
+import os
 from copy import copy
+from urllib.parse import urlparse
 
 from voluptuous import Any
 
 import dvc.prompt as prompt
 from dvc.cache import NamedCache
-from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
-from dvc.exceptions import DvcException
-from dvc.remote.base import RemoteBASE
+from dvc.exceptions import (
+    CollectCacheError,
+    DvcException,
+    RemoteCacheRequiredError,
+)
 
+from ..tree.base import BaseTree
 
 logger = logging.getLogger(__name__)
 
 
 class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = "output '{}' does not exist".format(path)
+        msg = f"output '{path}' does not exist"
         super().__init__(msg)
 
 
 class OutputIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = "output '{}' is not a file or directory".format(path)
+        msg = f"output '{path}' is not a file or directory"
         super().__init__(msg)
 
 
 class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
-        msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
+        msg = f""" output '{path}' is already tracked by SCM (e.g. Git).
+    You can remove it from Git, then add to DVC.
+        To stop tracking from Git:
+            git rm -r --cached '{path}'
+            git commit -m "stop tracking {path}" """
         super().__init__(msg)
 
 
 class OutputIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__("Stage file '{}' cannot be an output.".format(path))
+        super().__init__(f"Stage file '{path}' cannot be an output.")
 
 
-class OutputBase(object):
+class BaseOutput:
     IS_DEPENDENCY = False
 
-    REMOTE = RemoteBASE
+    TREE_CLS = BaseTree
 
     PARAM_PATH = "path"
     PARAM_CACHE = "cache"
     PARAM_METRIC = "metric"
     PARAM_METRIC_TYPE = "type"
     PARAM_METRIC_XPATH = "xpath"
+    PARAM_PLOT = "plot"
+    PARAM_PLOT_TEMPLATE = "template"
+    PARAM_PLOT_X = "x"
+    PARAM_PLOT_Y = "y"
+    PARAM_PLOT_X_LABEL = "x_label"
+    PARAM_PLOT_Y_LABEL = "y_label"
+    PARAM_PLOT_TITLE = "title"
+    PARAM_PLOT_HEADER = "header"
     PARAM_PERSIST = "persist"
 
     METRIC_SCHEMA = Any(
@@ -58,8 +74,6 @@ class OutputBase(object):
         },
     )
 
-    PARAM_TAGS = "tags"
-
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
     IsStageFileError = OutputIsStageFileError
@@ -71,11 +85,11 @@ class OutputBase(object):
         stage,
         path,
         info=None,
-        remote=None,
+        tree=None,
         cache=True,
         metric=False,
+        plot=False,
         persist=False,
-        tags=None,
     ):
         self._validate_output_path(path)
         # This output (and dependency) objects have too many paths/urls
@@ -92,21 +106,24 @@ class OutputBase(object):
         self.repo = stage.repo if stage else None
         self.def_path = path
         self.info = info
-        self.remote = remote or self.REMOTE(self.repo, {})
+        if tree:
+            self.tree = tree
+        else:
+            self.tree = self.TREE_CLS(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
+        self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
-        self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
-        self.path_info = self._parse_path(remote, path)
+        self.path_info = self._parse_path(tree, path)
         if self.use_cache and self.cache is None:
             raise RemoteCacheRequiredError(self.path_info)
 
-    def _parse_path(self, remote, path):
-        if remote:
+    def _parse_path(self, tree, path):
+        if tree:
             parsed = urlparse(path)
-            return remote.path_info / parsed.path.lstrip("/")
-        return self.REMOTE.path_cls(path)
+            return tree.path_info / parsed.path.lstrip("/")
+        return self.TREE_CLS.PATH_CLS(path)
 
     def __repr__(self):
         return "{class_name}: '{def_path}'".format(
@@ -118,7 +135,7 @@ class OutputBase(object):
 
     @property
     def scheme(self):
-        return self.REMOTE.scheme
+        return self.TREE_CLS.scheme
 
     @property
     def is_in_repo(self):
@@ -141,37 +158,48 @@ class OutputBase(object):
 
     @classmethod
     def supported(cls, url):
-        return cls.REMOTE.supported(url)
+        return cls.TREE_CLS.supported(url)
 
     @property
     def cache_path(self):
-        return self.cache.checksum_to_path_info(self.checksum).url
+        return self.cache.tree.hash_to_path_info(self.checksum).url
+
+    @property
+    def checksum_type(self):
+        return self.tree.PARAM_CHECKSUM
 
     @property
     def checksum(self):
-        return self.info.get(self.remote.PARAM_CHECKSUM)
+        return self.info.get(self.tree.PARAM_CHECKSUM)
+
+    @checksum.setter
+    def checksum(self, checksum):
+        self.info[self.tree.PARAM_CHECKSUM] = checksum
+
+    def get_checksum(self):
+        return self.tree.get_hash(self.path_info)
 
     @property
     def is_dir_checksum(self):
-        return self.remote.is_dir_checksum(self.checksum)
+        return self.tree.is_dir_hash(self.checksum)
 
     @property
     def exists(self):
-        return self.remote.exists(self.path_info)
+        return self.tree.exists(self.path_info)
+
+    def save_info(self):
+        return self.tree.save_info(self.path_info)
 
     def changed_checksum(self):
-        return self.checksum != self.remote.get_checksum(self.path_info)
+        return self.checksum != self.get_checksum()
 
-    def changed_cache(self):
+    def changed_cache(self, filter_info=None):
         if not self.use_cache or not self.checksum:
             return True
 
-        return self.cache.changed_cache(self.checksum)
-
-    def status(self):
-        if self.checksum and self.use_cache and self.changed_cache():
-            return {str(self): "not in cache"}
+        return self.cache.changed_cache(self.checksum, filter_info=filter_info)
 
+    def workspace_status(self):
         if not self.exists:
             return {str(self): "deleted"}
 
@@ -183,6 +211,12 @@ class OutputBase(object):
 
         return {}
 
+    def status(self):
+        if self.checksum and self.use_cache and self.changed_cache():
+            return {str(self): "not in cache"}
+
+        return self.workspace_status()
+
     def changed(self):
         status = self.status()
         logger.debug(str(status))
@@ -190,13 +224,32 @@ class OutputBase(object):
 
     @property
     def is_empty(self):
-        return self.remote.is_empty(self.path_info)
+        return self.tree.is_empty(self.path_info)
 
     def isdir(self):
-        return self.remote.isdir(self.path_info)
+        return self.tree.isdir(self.path_info)
 
     def isfile(self):
-        return self.remote.isfile(self.path_info)
+        return self.tree.isfile(self.path_info)
+
+    # pylint: disable=no-member
+
+    def ignore(self):
+        if not self.use_scm_ignore:
+            return
+
+        if self.repo.scm.is_tracked(self.fspath):
+            raise OutputAlreadyTrackedError(self)
+
+        self.repo.scm.ignore(self.fspath)
+
+    def ignore_remove(self):
+        if not self.use_scm_ignore:
+            return
+
+        self.repo.scm.ignore_remove(self.fspath)
+
+    # pylint: enable=no-member
 
     def save(self):
         if not self.exists:
@@ -206,39 +259,33 @@ class OutputBase(object):
             raise self.IsNotFileOrDirError(self)
 
         if self.is_empty:
-            logger.warning("'{}' is empty.".format(self))
+            logger.warning(f"'{self}' is empty.")
 
-        if self.use_scm_ignore:
-            if self.repo.scm.is_tracked(self.fspath):
-                raise OutputAlreadyTrackedError(self)
+        self.ignore()
 
-            self.repo.scm.ignore(self.fspath)
+        if self.metric or self.plot:
+            self.verify_metric()
 
         if not self.use_cache:
-            self.info = self.remote.save_info(self.path_info)
-            if self.metric:
-                self.verify_metric()
+            self.info = self.save_info()
             if not self.IS_DEPENDENCY:
-                logger.info(
-                    "Output '{}' doesn't use cache. Skipping saving.".format(
-                        self
-                    )
+                logger.debug(
+                    "Output '%s' doesn't use cache. Skipping saving.", self
                 )
             return
 
         assert not self.IS_DEPENDENCY
 
         if not self.changed():
-            logger.info(
-                "Output '{}' didn't change. Skipping saving.".format(self)
-            )
+            logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
-        self.info = self.remote.save_info(self.path_info)
+        self.info = self.save_info()
 
     def commit(self):
+        assert self.info
         if self.use_cache:
-            self.cache.save(self.path_info, self.info)
+            self.cache.save(self.path_info, self.cache.tree, self.info)
 
     def dumpd(self):
         ret = copy(self.info)
@@ -247,7 +294,8 @@ class OutputBase(object):
         if self.IS_DEPENDENCY:
             return ret
 
-        ret[self.PARAM_CACHE] = self.use_cache
+        if not self.use_cache:
+            ret[self.PARAM_CACHE] = self.use_cache
 
         if isinstance(self.metric, dict):
             if (
@@ -256,27 +304,27 @@ class OutputBase(object):
             ):
                 del self.metric[self.PARAM_METRIC_XPATH]
 
-        ret[self.PARAM_METRIC] = self.metric
-        ret[self.PARAM_PERSIST] = self.persist
+        if self.metric:
+            ret[self.PARAM_METRIC] = self.metric
 
-        if self.tags:
-            ret[self.PARAM_TAGS] = self.tags
+        if self.plot:
+            ret[self.PARAM_PLOT] = self.plot
+
+        if self.persist:
+            ret[self.PARAM_PERSIST] = self.persist
 
         return ret
 
     def verify_metric(self):
-        raise DvcException(
-            "verify metric is not supported for {}".format(self.scheme)
-        )
+        raise DvcException(f"verify metric is not supported for {self.scheme}")
 
     def download(self, to):
-        self.remote.download(self.path_info, to.path_info)
+        self.tree.download(self.path_info, to.path_info)
 
     def checkout(
         self,
         force=False,
         progress_callback=None,
-        tag=None,
         relink=False,
         filter_info=None,
     ):
@@ -287,14 +335,9 @@ class OutputBase(object):
                 )
             return None
 
-        if tag:
-            info = self.tags[tag]
-        else:
-            info = self.info
-
         return self.cache.checkout(
             self.path_info,
-            info,
+            self.info,
             force=force,
             progress_callback=progress_callback,
             relink=relink,
@@ -302,18 +345,19 @@ class OutputBase(object):
         )
 
     def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        self.tree.remove(self.path_info)
         if self.scheme != "local":
             return
 
-        if ignore_remove and self.use_scm_ignore:
-            self.repo.scm.ignore_remove(self.fspath)
+        if ignore_remove:
+            self.ignore_remove()
 
     def move(self, out):
+        # pylint: disable=no-member
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore_remove(self.fspath)
 
-        self.remote.move(self.path_info, out.path_info)
+        self.tree.move(self.path_info, out.path_info)
         self.def_path = out.def_path
         self.path_info = out.path_info
         self.save()
@@ -332,9 +376,20 @@ class OutputBase(object):
 
     def unprotect(self):
         if self.exists:
-            self.remote.unprotect(self.path_info)
+            self.tree.unprotect(self.path_info)
 
-    def _collect_used_dir_cache(
+    def get_dir_cache(self, **kwargs):
+        if not self.is_dir_checksum:
+            raise DvcException("cannot get dir cache for file checksum")
+        if self.cache.changed_cache_file(self.checksum):
+            self.repo.cloud.pull(
+                NamedCache.make("local", self.checksum, str(self)),
+                show_checksums=False,
+                **kwargs,
+            )
+        return self.dir_cache
+
+    def collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
     ):
         """Get a list of `info`s related to the given directory.
@@ -358,16 +413,10 @@ class OutputBase(object):
 
         cache = NamedCache()
 
-        if self.cache.changed_cache_file(self.checksum):
-            try:
-                self.repo.cloud.pull(
-                    NamedCache.make("local", self.checksum, str(self)),
-                    jobs=jobs,
-                    remote=remote,
-                    show_checksums=False,
-                )
-            except DvcException:
-                logger.debug("failed to pull cache for '{}'".format(self))
+        try:
+            self.get_dir_cache(jobs=jobs, remote=remote)
+        except DvcException:
+            logger.debug(f"failed to pull cache for '{self}'")
 
         if self.cache.changed_cache_file(self.checksum):
             msg = (
@@ -383,11 +432,21 @@ class OutputBase(object):
             else:
                 return cache
 
+        path = str(self.path_info)
+        filter_path = str(filter_info) if filter_info else None
+        is_win = os.name == "nt"
         for entry in self.dir_cache:
-            checksum = entry[self.remote.PARAM_CHECKSUM]
-            info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            if not filter_info or info.isin_or_eq(filter_info):
-                cache.add(self.scheme, checksum, str(info))
+            checksum = entry[self.tree.PARAM_CHECKSUM]
+            entry_relpath = entry[self.tree.PARAM_RELPATH]
+            if is_win:
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            entry_path = os.path.join(path, entry_relpath)
+            if (
+                not filter_path
+                or entry_path == filter_path
+                or entry_path.startswith(filter_path + os.sep)
+            ):
+                cache.add(self.scheme, checksum, entry_path)
 
         return cache
 
@@ -405,16 +464,27 @@ class OutputBase(object):
 
         if self.stage.is_repo_import:
             cache = NamedCache()
-            dep, = self.stage.deps
+            (dep,) = self.stage.deps
             cache.external[dep.repo_pair].add(dep.def_path)
             return cache
 
-        if not self.info:
-            logger.warning(
-                "Output '{}'({}) is missing version info. Cache for it will "
-                "not be collected. Use dvc repro to get your pipeline up to "
-                "date.".format(self, self.stage)
+        if not self.checksum:
+            msg = (
+                "Output '{}'({}) is missing version info. "
+                "Cache for it will not be collected. "
+                "Use `dvc repro` to get your pipeline up to date.".format(
+                    self, self.stage
+                )
             )
+            if self.exists:
+                msg += (
+                    "\n"
+                    "You can also use `dvc commit {stage.addressing}` "
+                    "to associate existing '{out}' with {stage}.".format(
+                        out=self, stage=self.stage
+                    )
+                )
+            logger.warning(msg)
             return NamedCache()
 
         ret = NamedCache.make(self.scheme, self.checksum, str(self))
@@ -422,13 +492,15 @@ class OutputBase(object):
         if not self.is_dir_checksum:
             return ret
 
-        ret.update(self._collect_used_dir_cache(**kwargs))
+        ret.add_child_cache(
+            self.checksum, self.collect_used_dir_cache(**kwargs),
+        )
 
         return ret
 
     @classmethod
     def _validate_output_path(cls, path):
-        from dvc.stage import Stage
+        from dvc.dvcfile import is_valid_filename
 
-        if Stage.is_valid_filename(path):
+        if is_valid_filename(path):
             raise cls.IsStageFileError(path)

dvc/output/gs.py

@@ -1,6 +1,7 @@
-from dvc.output.s3 import OutputS3
-from dvc.remote.gs import RemoteGS
+from dvc.output.s3 import S3Output
+
+from ..tree.gs import GSTree
 
 
-class OutputGS(OutputS3):
-    REMOTE = RemoteGS
+class GSOutput(S3Output):
+    TREE_CLS = GSTree

dvc/output/hdfs.py

@@ -1,6 +1,7 @@
-from dvc.output.base import OutputBase
-from dvc.remote.hdfs import RemoteHDFS
+from dvc.output.base import BaseOutput
+
+from ..tree.hdfs import HDFSTree
 
 
-class OutputHDFS(OutputBase):
-    REMOTE = RemoteHDFS
+class HDFSOutput(BaseOutput):
+    TREE_CLS = HDFSTree

dvc/output/local.py

@@ -4,24 +4,35 @@ from urllib.parse import urlparse
 
 from dvc.exceptions import DvcException
 from dvc.istextfile import istextfile
-from dvc.output.base import OutputBase
-from dvc.remote.local import RemoteLOCAL
+from dvc.output.base import BaseOutput
 from dvc.utils import relpath
-from dvc.compat import fspath_py35
 from dvc.utils.fs import path_isin
 
+from ..tree.local import LocalTree
 
 logger = logging.getLogger(__name__)
 
 
-class OutputLOCAL(OutputBase):
-    REMOTE = RemoteLOCAL
+class LocalOutput(BaseOutput):
+    TREE_CLS = LocalTree
     sep = os.sep
 
-    def _parse_path(self, remote, path):
+    def __init__(self, stage, path, *args, **kwargs):
+        if stage and path_isin(path, stage.repo.root_dir):
+            path = relpath(path, stage.wdir)
+
+        super().__init__(stage, path, *args, **kwargs)
+        if (
+            self.is_in_repo
+            and self.repo
+            and isinstance(self.repo.tree, LocalTree)
+        ):
+            self.tree = self.repo.tree
+
+    def _parse_path(self, tree, path):
         parsed = urlparse(path)
         if parsed.scheme == "remote":
-            p = remote.path_info / parsed.path.lstrip("/")
+            p = tree.path_info / parsed.path.lstrip("/")
         else:
             # NOTE: we can path either from command line or .dvc file,
             # so we should expect both posix and windows style paths.
@@ -29,15 +40,15 @@ class OutputLOCAL(OutputBase):
             #
             # FIXME: if we have Windows path containing / or posix one with \
             # then we have #2059 bug and can't really handle that.
-            p = self.REMOTE.path_cls(path)
-            if not p.is_absolute():
+            p = self.TREE_CLS.PATH_CLS(path)
+            if self.stage and not p.is_absolute():
                 p = self.stage.wdir / p
 
-        abs_p = os.path.abspath(os.path.normpath(fspath_py35(p)))
-        return self.REMOTE.path_cls(abs_p)
+        abs_p = os.path.abspath(os.path.normpath(p))
+        return self.TREE_CLS.PATH_CLS(abs_p)
 
     def __str__(self):
-        if not self.is_in_repo:
+        if not self.repo or not self.is_in_repo:
             return str(self.def_path)
 
         cur_dir = os.getcwd()
@@ -67,17 +78,18 @@ class OutputLOCAL(OutputBase):
         return ret
 
     def verify_metric(self):
-        if not self.metric:
+        if not self.metric or self.plot:
             return
 
-        path = fspath_py35(self.path_info)
+        path = os.fspath(self.path_info)
         if not os.path.exists(path):
             return
 
+        name = "metrics" if self.metric else "plot"
         if os.path.isdir(path):
-            msg = "directory '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.path_info))
+            msg = "directory '{}' cannot be used as {}."
+            raise IsADirectoryError(msg.format(self.path_info, name))
 
         if not istextfile(path):
-            msg = "binary file '{}' cannot be used as metrics."
-            raise DvcException(msg.format(self.path_info))
+            msg = "binary file '{}' cannot be used as {}."
+            raise DvcException(msg.format(self.path_info, name))

dvc/output/s3.py

@@ -1,6 +1,7 @@
-from dvc.output.base import OutputBase
-from dvc.remote.s3 import RemoteS3
+from dvc.output.base import BaseOutput
+
+from ..tree.s3 import S3Tree
 
 
-class OutputS3(OutputBase):
-    REMOTE = RemoteS3
+class S3Output(BaseOutput):
+    TREE_CLS = S3Tree