dvc: bump to 1.7.1

* tree: validate resolved remote:// sections

* dvc: use checksum_jobs instead of hash_jobs

This is a regression introduced during one of the refactorings.

.codeclimate.yml

@@ -1,5 +0,0 @@
-languages:
-    Python: true
-
-exclude_paths:
-    - tests/

.deepsource.toml

@@ -1,15 +0,0 @@
-version = 1
-
-test_patterns = [
-  'tests/**'
-]
-
-exclude_patterns = [
-  'dvc/utils/compat.py'
-]
-
-[[analyzers]]
-name = 'python'
-enabled = true
-runtime_version = '3.x.x'
-

.dvc/.gitignore

@@ -9,3 +9,4 @@
 /pkg
 /repos
 /tmp
+/experiments

.flake8

@@ -1,7 +0,0 @@
-[flake8]
-ignore =
-    E203, # Whitespace before ':'
-    E266, # Too many leading '#' for block comment
-    W503, # Line break occurred before a binary operator
-max-line-length = 79
-select = B,C,E,F,W,T4,B9

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+patreon: DVCorg

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,10 +1,18 @@
 ---
-name: Bug report
-about: Create a report to help us improve
-
+name: "\U0001F41B Bug Report"
+about: Create a bug report to help us improve DVC
 ---
 
+## Bug Report
 
-**Please provide information about your setup**
-DVC version(i.e. `dvc --version`), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))
+### Please provide information about your setup
 
+**Output of `dvc version`:**
+
+```console
+$ dvc version
+```
+
+**Additional Information (if any):**
+
+If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: "🤗 Need help?"
+    url: https://dvc.org/chat
+    about: If you have a question, ask us on Discord. Please join with this invite 👉

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,5 @@
 ---
-name: Feature request
+name: "\U0001F680 Feature Request"
 about: Suggest an idea for this project
-
 ---
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,5 @@
-* [ ] ❗ Have you followed the guidelines in the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) list?
+* [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
 
-* [ ] 📖 Check this box if this PR **does not** require [documentation](https://dvc.org/doc) updates, or if it does **and** you have created a separate PR in [dvc.org](https://github.com/iterative/dvc.org) with such updates (or at least opened an issue about it in that repo). Please link below to your PR (or issue) in the [dvc.org](https://github.com/iterative/dvc.org) repo.
-
-* [ ] ❌ Have you checked DeepSource, CodeClimate, and other sanity checks below? We consider their findings recommendatory and don't expect everything to be addressed. Please review them carefully and fix those that actually improve code or fix bugs.
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
 
 Thank you for the contribution - we'll try to review it as soon as possible. 🙏

.github/release-drafter.yml

@@ -1,5 +1,48 @@
 # Config for https://github.com/apps/release-drafter
-branches:
-  - master
+name-template: "$RESOLVED_VERSION 🦉 "
+tag-template: "$RESOLVED_VERSION"
+categories:
+  - title: "🚀 New Features and Enhancements "
+    labels:
+      - "feature"
+      - "enhancement"
+  - title: "💣 Breaking Changes "
+    labels:
+      - "breaking-change"
+  - title: "🏇 Optimizations "
+    labels:
+      - "optimize"
+      - "performance"
+  - title: "🐛 Bug Fixes "
+    labels:
+      - "bugfix"
+      - "bug"
+  - title: "🔨 Maintenance "
+    labels:
+      - "maintenance"
+      - "refactoring"
+      - "chore"
+      - "build"
+      - "ci"
+change-template: "- $TITLE (#$NUMBER) @$AUTHOR"
+version-resolver:
+  major:
+    labels:
+      - "major"
+  minor:
+    labels:
+      - "minor"
+  patch:
+    labels:
+      - "patch"
+  default: patch
+exclude-labels:
+  - "skip-changelog"
 template: |
+  > Refer to https://dvc.org/doc/install for installation instructions.
+
+  ## Changes
+
   $CHANGES
+
+  Thanks again to $CONTRIBUTORS for the contributions! 🎉

.github/workflows/check-patch.yaml

@@ -0,0 +1,19 @@
+name: Lint Python
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install requirements
+      run: bash ./scripts/ci/install.sh
+    - name: Check Patch
+      run: ./scripts/ci/check_patch.sh

.gitignore

@@ -1,12 +1,11 @@
 neatlynx.conf
 neatlynx/__pycache__
 .idea
-cache
+/cache
 *.pyc
 .env/
 .env2.7/
 
-cache
 .dvc.conf.lock
 .DS_Store
 build/

.pre-commit-config.yaml

@@ -1,17 +1,71 @@
 repos:
--   repo: https://github.com/ambv/black
+  - hooks:
+      - id: black
+        language_version: python3
+    repo: https://github.com/ambv/black
     rev: 19.10b0
+  - hooks:
+      - id: isort
+        language_version: python3
+    repo: https://github.com/timothycrosley/isort
+    rev: 5.4.2
+  - hooks:
+      - id: flake8
+        language_version: python3
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-debugger
+          - flake8-string-format
+    repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.3
+  - repo: local
     hooks:
-    - id: black
-      language_version: python3
--   repo: https://gitlab.com/pycqa/flake8
+    - id: pylint
+      name: pylint
+      entry: pylint
+      language: system
+      types: [python]
+  - hooks:
+      - args:
+          - -i
+          - "2"
+        id: beautysh
+        language_version: python3
+    repo: https://github.com/lovesegfault/beautysh
     rev: master
-    hooks:
-    - id: flake8
-      language_version: python3
--   repo: https://github.com/lovesegfault/beautysh
-    rev: master
-    hooks:
-    - id: beautysh
-      language_version: python3
-      args: [-i, '2']  # 2-space indentaion
+  - hooks:
+      - args:
+          - git-hook
+          - pre-commit
+        entry: dvc
+        id: dvc-pre-commit
+        language: system
+        name: DVC pre-commit
+        stages:
+          - commit
+        verbose: true
+        require_serial: true
+      - args:
+          - git-hook
+          - pre-push
+        entry: dvc
+        id: dvc-pre-push
+        language: system
+        name: DVC pre-push
+        stages:
+          - push
+        require_serial: true
+      - always_run: true
+        args:
+          - git-hook
+          - post-checkout
+        entry: dvc
+        id: dvc-post-checkout
+        language: system
+        minimum_pre_commit_version: 2.2.0
+        name: DVC post-checkout
+        stages:
+          - post-checkout
+        require_serial: true
+    repo: local

.pre-commit-hooks.yaml

@@ -0,0 +1,36 @@
+- args:
+  - git-hook
+  - pre-commit
+  entry: dvc
+  id: dvc-pre-commit
+  language: python
+  language_version: python3
+  name: DVC pre-commit
+  require_serial: true
+  stages:
+  - commit
+  verbose: true
+- args:
+  - git-hook
+  - pre-push
+  entry: dvc
+  id: dvc-pre-push
+  language: python
+  language_version: python3
+  name: DVC pre-push
+  require_serial: true
+  stages:
+  - push
+- always_run: true
+  args:
+  - git-hook
+  - post-checkout
+  entry: dvc
+  id: dvc-post-checkout
+  language: python
+  language_version: python3
+  minimum_pre_commit_version: 2.2.0
+  name: DVC post-checkout
+  require_serial: true
+  stages:
+  - post-checkout

.pylintrc

@@ -15,7 +15,11 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-#init-hook=
+init-hook="import sys; \
+           from pathlib import Path; \
+           from pylint.config import find_pylintrc; \
+           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
@@ -28,7 +32,7 @@ limit-inference-results=100
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
-load-plugins=
+load-plugins=pylint_pytest,pylint_plugin_disable
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -60,7 +64,16 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use "--disable=all --enable=classes
 # --disable=W".
-disable=print-statement,
+disable=format,
+        # ^ most of them are covered by flake8,
+        # and are conventions rather than warnings or errors
+        refactoring,  # will be noisy if it starts giving suggestions
+        spelling,
+        # design is too-few-* and too-many-* checks, too-much-* for us right now
+        design,
+        # following were generated by default
+        # most of them are redundant Py3 checks for Py2+3 codebase
+        print-statement,
         parameter-unpacking,
         unpacking-in-except,
         old-raise-syntax,
@@ -139,9 +152,33 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        useless-object-inheritance,
-        missing-docstring,
-        bad-continuation
+        # custom disables start here
+        no-self-use,
+        invalid-name,
+        # already a check enabled for `wildcard-import`
+        unused-wildcard-import,
+        misplaced-comparison-constant,
+        # probably soon: https://github.com/iterative/dvc/issues/1843
+        logging-format-interpolation,
+        logging-fstring-interpolation,
+        # we don't have proper docstrings yet
+        # even if we do, we'll be doing it gradually
+        missing-function-docstring,
+        missing-module-docstring,
+        missing-class-docstring,
+        duplicate-code,
+        # too noisy, and detects issues even when import is inside a function
+        # see: https://github.com/PyCQA/pylint/issues/850
+        cyclic-import,
+        # isort should cover these:
+        wrong-import-order,
+        wrong-import-position,
+        ungrouped-imports,
+        multiple-imports,
+        # we use it to speedup/optimize
+        import-outside-toplevel,
+        fixme,
+
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -228,7 +265,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -249,7 +286,7 @@ ignore-on-opaque-inference=yes
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
+ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
@@ -290,7 +327,7 @@ dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore.
-ignored-argument-names=_.*|^ignored_|^unused_
+ignored-argument-names=_.*|^ignored_|^unused_|args|kwargs
 
 # Tells whether we should check for unused import in __init__ files.
 init-import=no

.restyled.yaml

@@ -9,3 +9,12 @@ restylers:
     - "**/*.py"
     interpreters:
     - python
+  - name: isort
+    image: restyled/restyler-isort:v5.4.2
+    command:
+    - isort
+    arguments: []
+    include:
+    - "**/*.py"
+    interpreters:
+    - python

.travis.yml

@@ -29,7 +29,7 @@ jobs:
     os: linux
     language: python
     python: 3.7
-    before_install:
+    before_install: bash ./scripts/ci/install.sh
     install:
     script: ./scripts/ci/check_patch.sh
   # test jobs
@@ -37,15 +37,24 @@ jobs:
     stage: test
     os: windows           # Windows 10.0.17134 N/A Build 17134
     language: shell       # 'language: python' is an error on Travis CI Windows
-  - name: "3.5 on Linux"
-    language: python
-    python: 3.5
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
+  - name: "3.8 on Windows"
+    os: windows           # Windows 10.0.17134 N/A Build 17134
+    language: shell       # 'language: python' is an error on Travis CI Windows
+    env:
+      - PYTHON_VERSION=3.8.0
+      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
   - name: "3.6 on Linux"
     language: python
     python: 3.6
   - name: "3.7 on Linux"
     language: python
     python: 3.7
+  - name: "3.8 on Linux"
+    language: python
+    python: 3.8
   - name: "3.7 on Mac"
     os: osx
     osx_image: xcode11.3
@@ -63,6 +72,9 @@ jobs:
     language: shell
     install:
     script: ./scripts/build_windows.cmd
+    env:
+      - PYTHON_VERSION=3.7.5
+      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
   - name: "Linux pkgs"
     language: python
     python: 3.7
@@ -71,11 +83,11 @@ jobs:
     script: ./scripts/build_posix.sh
   - name: "PyPI pkgs"
     language: python
-    python: 3.7
+    python: 3.8
     script: ./scripts/build_package.sh
   - name: Snapcraft snap
     language: python
-    python: 3.7
+    python: 3.8
     addons:
      snaps:
      - name: snapcraft
@@ -86,8 +98,12 @@ jobs:
      - name: lxd
        channel: stable
     env:
-    - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}'
+    - SNAPCRAFT_IMAGE_INFO: |
+        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
+    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
+    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
     install:
+    - sudo usermod --append --groups lxd $USER
     - sudo /snap/bin/lxd.migrate -yes
     - sudo /snap/bin/lxd waitready
     - sudo /snap/bin/lxd init --auto
@@ -136,12 +152,11 @@ deploy:
       tags: true
       repo: iterative/dvc
       stage: build
-  - provider: snap
-    snap: dvc_*.snap
-    channel: $SNAP_CHANNEL
+  - provider: script
     skip_cleanup: true
+    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
     on:
       all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)"
+      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
       repo: iterative/dvc
       stage: build

.zenodo.json

@@ -0,0 +1,8 @@
+{
+  "title": "DVC: Data Version Control - Git for Data & Models",
+  "keywords": [
+    "data-science", "data-version-control", "machine-learning", "git",
+    "developer-tools", "reproducibility", "collaboration", "ai", "python"],
+  "contributors": [
+    {"name": "DVC team", "type": "Other", "affiliation": "Iterative"}]
+}

README.rst

@@ -1,6 +1,4 @@
-.. image:: https://dvc.org/static/img/logo-github-readme.png
-  :target: https://dvc.org
-  :alt: DVC logo
+|Banner|
 
 `Website <https://dvc.org>`_
 • `Docs <https://dvc.org/doc>`_
@@ -10,33 +8,9 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-.. image:: https://img.shields.io/badge/release-ok-brightgreen
-  :target: https://travis-ci.com/iterative/dvc
-  :alt: Release
+|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
 
-.. image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev
-  :target: https://travis-ci.com/iterative/dvc
-  :alt: Travis dev branch
-
-.. image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
-  :target: https://codeclimate.com/github/iterative/dvc
-  :alt: Code Climate
-
-.. image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
-  :target: https://codecov.io/gh/iterative/dvc
-  :alt: Codecov
-
-.. image:: https://img.shields.io/badge/patreon-donate-green.svg
-  :target: https://www.patreon.com/DVCorg/overview
-  :alt: Donate
-
-.. image:: https://anaconda.org/conda-forge/dvc/badges/version.svg
-  :target: https://anaconda.org/conda-forge/dvc
-  :alt: Conda-forge
-
-.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
-  :target: https://snapcraft.io/dvc
-  :alt: Snapcraft
+|PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
 
 |
 
@@ -79,9 +53,7 @@ to store data and model files seamlessly out of Git, while preserving almost the
 were stored in Git itself. To store and share the data cache, DVC supports multiple remotes - any cloud (S3, Azure,
 Google Cloud, etc) or any on-premise network storage (via SSH, for example).
 
-.. image:: https://dvc.org/static/img/flow.gif
-   :target: https://dvc.org/static/img/flow.gif
-   :alt: how_dvc_works
+|Flowchart|
 
 The DVC pipelines (computational graph) feature connects code and data together. It is possible to explicitly
 specify all steps required to produce a model: input dependencies including data, commands to run,
@@ -119,9 +91,51 @@ Installation
 There are four options to install DVC: ``pip``, Homebrew, Conda (Anaconda) or an OS-specific package.
 Full instructions are `available here <https://dvc.org/doc/get-started/install>`_.
 
+Snap (Snapcraft/Linux)
+----------------------
+
+|Snap|
+
+.. code-block:: bash
+
+   snap install dvc --classic
+
+This corresponds to the latest tagged release.
+Add ``--beta`` for the latest tagged release candidate,
+or ``--edge`` for the latest ``master`` version.
+
+Choco (Chocolatey/Windows)
+--------------------------
+
+|Choco|
+
+.. code-block:: bash
+
+   choco install dvc
+
+Brew (Homebrew/Mac OS)
+----------------------
+
+|Brew|
+
+.. code-block:: bash
+
+   brew install dvc
+
+Conda (Anaconda)
+----------------
+
+|Conda|
+
+.. code-block:: bash
+
+   conda install -c conda-forge dvc
+
 pip (PyPI)
 ----------
 
+|PyPI|
+
 .. code-block:: bash
 
    pip install dvc
@@ -137,36 +151,11 @@ To install the development version, run:
 
    pip install git+git://github.com/iterative/dvc
 
-Homebrew
---------
-
-.. code-block:: bash
-
-   brew install dvc
-
-
-Conda (Anaconda)
-----------------
-
-.. code-block:: bash
-
-   conda install -c conda-forge dvc
-
-Currently, this includes support for Python versions 2.7, 3.6 and 3.7.
-
-Snap (Snapcraft)
-----------------
-
-.. code-block:: bash
-
-   snap install dvc --classic
-
-This corresponds to the latest tagged release.
-Add ``--edge`` for the latest ``master`` version.
-
 Package
 -------
 
+|Packages|
+
 Self-contained packages for Linux, Windows, and Mac are available. The latest version of the packages
 can be found on the GitHub `releases page <https://github.com/iterative/dvc/releases>`_.
 
@@ -206,40 +195,43 @@ Comparison to related technologies
 
 Contributing
 ============
+
+|Maintainability| |Donate|
+
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
-  :alt: 0
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+   :alt: 0
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
-  :alt: 1
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+   :alt: 1
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
-  :alt: 2
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+   :alt: 2
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
-  :alt: 3
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+   :alt: 3
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
-  :alt: 4
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+   :alt: 4
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
-  :alt: 5
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+   :alt: 5
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
-  :alt: 6
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+   :alt: 6
 
 .. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
-  :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
-  :alt: 7
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+   :alt: 7
 
 Mailing List
 ============
@@ -253,3 +245,67 @@ This project is distributed under the Apache license version 2.0 (see the LICENS
 
 By submitting a pull request to this project, you agree to license your contribution under the Apache license version
 2.0 to this project.
+
+Citation
+========
+
+|DOI|
+
+Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
+`DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
+
+.. |Banner| image:: https://dvc.org/img/logo-github-readme.png
+   :target: https://dvc.org
+   :alt: DVC logo
+
+.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
+   :target: https://travis-ci.com/iterative/dvc/branches
+   :alt: Release
+
+.. |CI| image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev&logo=travis
+   :target: https://travis-ci.com/iterative/dvc/builds
+   :alt: Travis dev branch
+
+.. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
+   :target: https://codeclimate.com/github/iterative/dvc
+   :alt: Code Climate
+
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/iterative/dvc
+   :alt: Codecov
+
+.. |Donate| image:: https://img.shields.io/badge/patreon-donate-green.svg?logo=patreon
+   :target: https://www.patreon.com/DVCorg/overview
+   :alt: Donate
+
+.. |Snap| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft
+   :target: https://snapcraft.io/dvc
+   :alt: Snapcraft
+
+.. |Choco| image:: https://img.shields.io/chocolatey/v/dvc?label=choco
+   :target: https://chocolatey.org/packages/dvc
+   :alt: Chocolatey
+
+.. |Brew| image:: https://img.shields.io/homebrew/v/dvc?label=brew
+   :target: https://formulae.brew.sh/formula/dvc
+   :alt: Homebrew
+
+.. |Conda| image:: https://img.shields.io/conda/v/conda-forge/dvc.svg?label=conda&logo=conda-forge
+   :target: https://anaconda.org/conda-forge/dvc
+   :alt: Conda-forge
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/dvc.svg?label=pip&logo=PyPI&logoColor=white
+   :target: https://pypi.org/project/dvc
+   :alt: PyPI
+
+.. |Packages| image:: https://img.shields.io/github/v/release/iterative/dvc?label=deb|pkg|rpm|exe&logo=GitHub
+   :target: https://github.com/iterative/dvc/releases/latest
+   :alt: deb|pkg|rpm|exe
+
+.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.3677553-blue.svg
+   :target: https://doi.org/10.5281/zenodo.3677553
+   :alt: DOI
+
+.. |Flowchart| image:: https://dvc.org/img/flow.gif
+   :target: https://dvc.org/img/flow.gif
+   :alt: how_dvc_works

dvc/__init__.py

@@ -3,17 +3,7 @@ DVC
 ----
 Make your data science projects reproducible and shareable.
 """
-import warnings
-
 import dvc.logger
 from dvc.version import __version__  # noqa: F401
 
-
 dvc.logger.setup()
-
-# Ignore numpy's runtime warnings: https://github.com/numpy/numpy/pull/432.
-# We don't directly import numpy, but our dependency networkx does, causing
-# these warnings in some environments. Luckily these warnings are benign and
-# we can simply ignore them so that they don't show up when you are using dvc.
-warnings.filterwarnings("ignore", message="numpy.dtype size changed")
-warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

dvc/analytics.py

@@ -2,12 +2,12 @@ import json
 import logging
 import os
 import platform
-import requests
 import sys
 import tempfile
 import uuid
 
 import distro
+import requests
 
 from dvc import __version__
 from dvc.config import Config, to_bool
@@ -15,11 +15,11 @@ from dvc.daemon import daemon
 from dvc.exceptions import NotDvcRepoError
 from dvc.lock import Lock, LockError
 from dvc.repo import Repo
-from dvc.scm import SCM
+from dvc.scm import SCM, NoSCM
+from dvc.scm.base import SCMError
 from dvc.utils import env2bool, is_binary
 from dvc.utils.fs import makedirs
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -35,7 +35,7 @@ def collect_and_send_report(args=None, return_code=None):
     report as a JSON, where the _collector_ generates it and the _sender_
     removes it after sending it.
     """
-    report = _runtime_info()
+    report = {}
 
     # Include command execution information on the report only when available.
     if args and hasattr(args, "func"):
@@ -61,7 +61,7 @@ def is_enabled():
     return enabled
 
 
-def send(report):
+def send(path):
     """
     Side effect: Removes the report after sending it.
 
@@ -72,19 +72,25 @@ def send(report):
     url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
-    with open(report, "rb") as fobj:
-        try:
-            requests.post(url, data=fobj, headers=headers, timeout=5)
-        except requests.exceptions.RequestException:
-            logger.debug("failed to send analytics report", exc_info=True)
+    with open(path) as fobj:
+        report = json.load(fobj)
 
-    os.remove(report)
+    report.update(_runtime_info())
+
+    try:
+        requests.post(url, json=report, headers=headers, timeout=5)
+    except requests.exceptions.RequestException:
+        logger.debug("failed to send analytics report", exc_info=True)
+
+    os.remove(path)
 
 
 def _scm_in_use():
     try:
         scm = SCM(root_dir=Repo.find_root())
         return type(scm).__name__
+    except SCMError:
+        return NoSCM.__name__
     except NotDvcRepoError:
         pass
 
@@ -152,7 +158,7 @@ def _find_or_create_user_id():
     try:
         with Lock(lockfile):
             try:
-                with open(fname, "r") as fobj:
+                with open(fname) as fobj:
                     user_id = json.load(fobj)["user_id"]
 
             except (FileNotFoundError, ValueError, KeyError):
@@ -164,6 +170,4 @@ def _find_or_create_user_id():
             return user_id
 
     except LockError:
-        logger.debug(
-            "Failed to acquire '{lockfile}'".format(lockfile=lockfile)
-        )
+        logger.debug(f"Failed to acquire '{lockfile}'")

dvc/api.py

@@ -1,37 +1,58 @@
 import os
-from contextlib import contextmanager, _GeneratorContextManager as GCM
+from contextlib import _GeneratorContextManager as GCM
+from contextlib import contextmanager
 
-from dvc.repo import Repo
-from dvc.exceptions import DvcException, NotDvcRepoError
+from funcy import reraise
+
+from dvc.exceptions import (
+    NotDvcRepoError,
+    OutputNotFoundError,
+    PathMissingError,
+)
 from dvc.external_repo import external_repo
-
-
-class UrlNotDvcRepoError(DvcException):
-    """Thrown if given URL is not a DVC repository.
-
-    Args:
-        url (str): URL to the repository
-    """
-
-    def __init__(self, url):
-        super().__init__("'{}' is not a DVC repository.".format(url))
+from dvc.path_info import PathInfo
+from dvc.repo import Repo
 
 
 def get_url(path, repo=None, rev=None, remote=None):
     """
-    Returns the full URL to the data artifact specified by its `path` in a
-    `repo`.
-    NOTE: There is no guarantee that the file actually exists in that location.
+    Returns the URL to the storage location of a data file or directory tracked
+    in a DVC repo. For Git repos, HEAD is used unless a rev argument is
+    supplied. The default remote is tried unless a remote argument is supplied.
+
+    Raises OutputNotFoundError if the file is not a dvc-tracked file.
+
+    NOTE: This function does not check for the actual existence of the file or
+    directory in the remote storage.
     """
     with _make_repo(repo, rev=rev) as _repo:
-        _require_dvc(_repo)
-        out = _repo.find_out_by_relpath(path)
-        remote_obj = _repo.cloud.get_remote(remote)
-        return str(remote_obj.checksum_to_path_info(out.checksum))
+        path_info = PathInfo(_repo.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, repo)):
+            metadata = _repo.repo_tree.metadata(path_info)
+
+        if not metadata.is_dvc:
+            raise OutputNotFoundError(path, repo)
+
+        cloud = metadata.repo.cloud
+        hash_info = _repo.repo_tree.get_hash(path_info)
+        return cloud.get_url_for(remote, checksum=hash_info.value)
 
 
-def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Context manager to open a file artifact as a file object."""
+def open(  # noqa, pylint: disable=redefined-builtin
+    path, repo=None, rev=None, remote=None, mode="r", encoding=None
+):
+    """
+    Open file in the supplied path tracked in a repo (both DVC projects and
+    plain Git repos are supported). For Git repos, HEAD is used unless a rev
+    argument is supplied. The default remote is tried unless a remote argument
+    is supplied. It may only be used as a context manager:
+
+        with dvc.api.open(
+                'path/to/file',
+                repo='https://example.com/url/to/repo'
+                ) as fd:
+            # ... Handle file object fd
+    """
     args = (path,)
     kwargs = {
         "repo": repo,
@@ -44,7 +65,9 @@ def open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
 
 
 class _OpenContextManager(GCM):
-    def __init__(self, func, args, kwds):
+    def __init__(
+        self, func, args, kwds
+    ):  # pylint: disable=super-init-not-called
         self.gen = func(*args, **kwds)
         self.func, self.args, self.kwds = func, args, kwds
 
@@ -63,7 +86,11 @@ def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
 
 
 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    """Returns the contents of a file artifact."""
+    """
+    Returns the contents of a tracked file (by DVC or Git). For Git repos, HEAD
+    is used unless a rev argument is supplied. The default remote is tried
+    unless a remote argument is supplied.
+    """
     with open(
         path, repo=repo, rev=rev, remote=remote, mode=mode, encoding=encoding
     ) as fd:
@@ -75,14 +102,9 @@ def _make_repo(repo_url=None, rev=None):
     repo_url = repo_url or os.getcwd()
     if rev is None and os.path.exists(repo_url):
         try:
-            yield Repo(repo_url)
+            yield Repo(repo_url, subrepos=True)
             return
         except NotDvcRepoError:
             pass  # fallthrough to external_repo
     with external_repo(url=repo_url, rev=rev) as repo:
         yield repo
-
-
-def _require_dvc(repo):
-    if not isinstance(repo, Repo):
-        raise UrlNotDvcRepoError(repo.url)

dvc/cache.py

@@ -1,104 +0,0 @@
-"""Manages cache of a DVC repo."""
-from collections import defaultdict
-
-from funcy import cached_property
-
-
-def _make_remote_property(name):
-    """
-    The config file is stored in a way that allows you to have a
-    cache for each remote.
-
-    This is needed when specifying external outputs
-    (as they require you to have an external cache location).
-
-    Imagine a config file like the following:
-
-            ['remote "dvc-storage"']
-            url = ssh://localhost/tmp
-            ask_password = true
-
-            [cache]
-            ssh = dvc-storage
-
-    This method creates a cached property, containing cache named `name`:
-
-        self.config == {'ssh': 'dvc-storage'}
-        self.ssh  # a RemoteSSH instance
-    """
-
-    def getter(self):
-        from dvc.remote import Remote
-
-        remote = self.config.get(name)
-        if not remote:
-            return None
-
-        return Remote(self.repo, name=remote)
-
-    getter.__name__ = name
-    return cached_property(getter)
-
-
-class Cache(object):
-    """Class that manages cache locations of a DVC repo.
-
-    Args:
-        repo (dvc.repo.Repo): repo instance that this cache belongs to.
-    """
-
-    CACHE_DIR = "cache"
-
-    def __init__(self, repo):
-        from dvc.remote import Remote
-
-        self.repo = repo
-        self.config = config = repo.config["cache"]
-
-        local = config.get("local")
-
-        if local:
-            settings = {"name": local}
-        else:
-            settings = {**config, "url": config["dir"]}
-
-        self.local = Remote(repo, **settings)
-
-    s3 = _make_remote_property("s3")
-    gs = _make_remote_property("gs")
-    ssh = _make_remote_property("ssh")
-    hdfs = _make_remote_property("hdfs")
-    azure = _make_remote_property("azure")
-
-
-class NamedCache(object):
-    def __init__(self):
-        self._items = defaultdict(lambda: defaultdict(set))
-        self.external = defaultdict(set)
-
-    @classmethod
-    def make(cls, scheme, checksum, name):
-        cache = cls()
-        cache.add(scheme, checksum, name)
-        return cache
-
-    def __getitem__(self, key):
-        return self._items[key]
-
-    def add(self, scheme, checksum, name):
-        self._items[scheme][checksum].add(name)
-
-    def add_external(self, url, rev, path):
-        self.external[url, rev].add(path)
-
-    def update(self, cache, suffix=""):
-        for scheme, src in cache._items.items():
-            dst = self._items[scheme]
-            for checksum, names in src.items():
-                if suffix:
-                    dst[checksum].update(n + suffix for n in names)
-                else:
-                    dst[checksum].update(names)
-
-        for repo_pair, files in cache.external.items():
-            self.external[repo_pair].update(files)

dvc/cache/__init__.py

@@ -0,0 +1,176 @@
+"""Manages cache of a DVC repo."""
+from collections import defaultdict
+
+from funcy import cached_property
+
+
+def _make_remote_property(name):
+    """
+    The config file is stored in a way that allows you to have a
+    cache for each remote.
+
+    This is needed when specifying external outputs
+    (as they require you to have an external cache location).
+
+    Imagine a config file like the following:
+
+            ['remote "dvc-storage"']
+            url = ssh://localhost/tmp
+            ask_password = true
+
+            [cache]
+            ssh = dvc-storage
+
+    This method creates a cached property, containing cache named `name`:
+
+        self.config == {'ssh': 'dvc-storage'}
+        self.ssh  # a RemoteSSH instance
+    """
+
+    def getter(self):
+        from ..tree import get_cloud_tree
+        from .base import CloudCache
+
+        remote = self.config.get(name)
+        if not remote:
+            return None
+
+        tree = get_cloud_tree(self.repo, name=remote)
+        return CloudCache(tree)
+
+    getter.__name__ = name
+    return cached_property(getter)
+
+
+class Cache:
+    """Class that manages cache locations of a DVC repo.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that this cache belongs to.
+    """
+
+    CACHE_DIR = "cache"
+
+    def __init__(self, repo):
+        from ..tree import get_cloud_tree
+        from .local import LocalCache
+
+        self.repo = repo
+        self.config = config = repo.config["cache"]
+
+        local = config.get("local")
+
+        if local:
+            settings = {"name": local}
+        else:
+            from ..config import LOCAL_COMMON
+
+            settings = {"url": config["dir"]}
+            for opt in LOCAL_COMMON.keys():
+                if opt in config:
+                    settings[str(opt)] = config.get(opt)
+
+        tree = get_cloud_tree(repo, **settings)
+        self.local = LocalCache(tree)
+
+    s3 = _make_remote_property("s3")
+    gs = _make_remote_property("gs")
+    ssh = _make_remote_property("ssh")
+    hdfs = _make_remote_property("hdfs")
+    azure = _make_remote_property("azure")
+
+
+class NamedCacheItem:
+    def __init__(self):
+        self.names = set()
+        self.children = defaultdict(NamedCacheItem)
+
+    def __eq__(self, other):
+        return self.names == other.names and self.children == other.children
+
+    def child_keys(self):
+        for key, child in self.children.items():
+            yield key
+            yield from child.child_keys()
+
+    def child_names(self):
+        for key, child in self.children.items():
+            yield key, child.names
+            yield from child.child_names()
+
+    def add(self, checksum, item):
+        self.children[checksum].update(item)
+
+    def update(self, item, suffix=""):
+        if suffix:
+            self.names.update(n + suffix for n in item.names)
+        else:
+            self.names.update(item.names)
+        for checksum, child_item in item.children.items():
+            self.children[checksum].update(child_item)
+
+
+class NamedCache:
+    # pylint: disable=protected-access
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
+        self.external = defaultdict(set)
+
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
+
+    def __getitem__(self, key):
+        return self._items[key]
+
+    def add(self, scheme, checksum, name):
+        """Add a mapped name for the specified checksum."""
+        self._items[scheme][checksum].names.add(name)
+
+    def add_child_cache(self, checksum, cache, suffix=""):
+        """Add/update child cache for the specified checksum."""
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme][checksum].children
+            for child_checksum, item in src.items():
+                dst[child_checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)
+
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
+
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, item in src.items():
+                dst[checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)
+
+    def scheme_keys(self, scheme):
+        """Iterate over a flat list of all keys for the specified scheme,
+        including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key
+            yield from item.child_keys()
+
+    def scheme_names(self, scheme):
+        """Iterate over a flat list of checksum, names items for the specified
+        scheme, including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key, item.names
+            yield from item.child_names()
+
+    def dir_keys(self, scheme):
+        return (
+            key for key, item in self._items[scheme].items() if item.children
+        )
+
+    def child_keys(self, scheme, checksum):
+        return self._items[scheme][checksum].child_keys()

dvc/cache/base.py

@@ -0,0 +1,694 @@
+import json
+import logging
+from copy import copy
+from operator import itemgetter
+
+from shortuuid import uuid
+
+import dvc.prompt as prompt
+from dvc.exceptions import (
+    CheckoutError,
+    ConfirmRemoveError,
+    DvcException,
+    MergeError,
+)
+from dvc.hash_info import HashInfo
+from dvc.path_info import WindowsPathInfo
+from dvc.progress import Tqdm
+from dvc.remote.slow_link_detection import slow_link_guard
+
+from ..tree.base import RemoteActionNotImplemented
+
+logger = logging.getLogger(__name__)
+
+STATUS_OK = 1
+STATUS_MISSING = 2
+STATUS_NEW = 3
+STATUS_DELETED = 4
+
+STATUS_MAP = {
+    # (local_exists, remote_exists)
+    (True, True): STATUS_OK,
+    (False, False): STATUS_MISSING,
+    (True, False): STATUS_NEW,
+    (False, True): STATUS_DELETED,
+}
+
+
+class DirCacheError(DvcException):
+    def __init__(self, hash_):
+        super().__init__(
+            f"Failed to load dir cache for hash value: '{hash_}'."
+        )
+
+
+class CloudCache:
+    """Cloud cache class."""
+
+    DEFAULT_CACHE_TYPES = ["copy"]
+    CACHE_MODE = None
+
+    def __init__(self, tree):
+        self.tree = tree
+        self.repo = tree.repo
+
+        self.cache_types = tree.config.get("type") or copy(
+            self.DEFAULT_CACHE_TYPES
+        )
+        self.cache_type_confirmed = False
+        self._dir_info = {}
+
+    def get_dir_cache(self, hash_info):
+        assert hash_info
+
+        dir_info = self._dir_info.get(hash_info.value)
+        if dir_info:
+            return dir_info
+
+        try:
+            dir_info = self.load_dir_cache(hash_info)
+        except DirCacheError:
+            dir_info = []
+
+        self._dir_info[hash_info.value] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, hash_info):
+        path_info = self.tree.hash_to_path_info(hash_info.value)
+
+        try:
+            with self.tree.open(path_info, "r") as fobj:
+                d = json.load(fobj)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(hash_info) from exc
+
+        if not isinstance(d, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                path_info,
+            )
+            return []
+
+        if self.tree.PATH_CLS == WindowsPathInfo:
+            # only need to convert it for Windows
+            for info in d:
+                # NOTE: here is a BUG, see comment to .as_posix() below
+                relpath = info[self.tree.PARAM_RELPATH]
+                info[self.tree.PARAM_RELPATH] = relpath.replace(
+                    "/", self.tree.PATH_CLS.sep
+                )
+
+        return d
+
+    def changed(self, path_info, hash_info):
+        """Checks if data has changed.
+
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
+            - There's no file in the cache
+
+        Args:
+            path_info: dict with path information.
+            hash: expected hash value for this data.
+
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug(
+            "checking if '%s'('%s') has changed.", path_info, hash_info
+        )
+
+        if not self.tree.exists(path_info):
+            logger.debug("'%s' doesn't exist.", path_info)
+            return True
+
+        if not hash_info:
+            logger.debug("hash value for '%s' is missing.", path_info)
+            return True
+
+        if self.changed_cache(hash_info):
+            logger.debug(
+                "cache for '%s'('%s') has changed.", path_info, hash_info
+            )
+            return True
+
+        actual = self.tree.get_hash(path_info)
+        if hash_info != actual:
+            logger.debug(
+                "hash value '%s' for '%s' has changed (actual '%s').",
+                hash_info,
+                actual,
+                path_info,
+            )
+            return True
+
+        logger.debug("'%s' hasn't changed.", path_info)
+        return False
+
+    def link(self, from_info, to_info):
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.tree.isfile(from_info)
+
+        self.tree.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self.tree, f"is_{link_type}", None)
+        if is_link and not is_link(path_info):
+            self.tree.remove(path_info)
+            raise DvcException(f"failed to verify {link_type}")
+
+        self.cache_type_confirmed = True
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self.tree, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self._verify_link(to_info, link_types[0])
+                return
+
+            except DvcException as exc:
+                logger.debug(
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
+                )
+                del link_types[0]
+
+        raise DvcException("no possible cache types left to try out.")
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.tree.exists(to_info):
+            raise DvcException(f"Link '{to_info}' already exists!")
+
+        link_method(from_info, to_info)
+
+        logger.debug(
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
+        )
+
+    def _save_file(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        assert hash_info
+
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
+        if tree == self.tree:
+            if self.changed_cache(hash_info):
+                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.link(cache_info, path_info)
+            elif self.tree.iscopy(path_info) and self._cache_is_copy(
+                path_info
+            ):
+                # Default relink procedure involves unneeded copy
+                self.tree.unprotect(path_info)
+            else:
+                self.tree.remove(path_info)
+                self.link(cache_info, path_info)
+
+            if save_link:
+                self.tree.state.save_link(path_info)
+            # we need to update path and cache, since in case of reflink,
+            # or copy cache type moving original file results in updates on
+            # next executed command, which causes md5 recalculation
+            self.tree.state.save(path_info, hash_info.value)
+        else:
+            if self.changed_cache(hash_info):
+                with tree.open(path_info, mode="rb") as fobj:
+                    # if tree has fetch enabled, DVC out will be fetched on
+                    # open and we do not need to read/copy any data
+                    if not (
+                        tree.isdvc(path_info, strict=False) and tree.fetch
+                    ):
+                        self.tree.copy_fobj(fobj, cache_info)
+                callback = kwargs.get("download_callback")
+                if callback:
+                    callback(1)
+
+        self.tree.state.save(cache_info, hash_info.value)
+
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.tree.path_info / ".cache_type_test_file"
+        if not self.tree.exists(test_cache_file):
+            with self.tree.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.tree.remove(workspace_file)
+            self.tree.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
+    def _get_dir_info_hash(self, dir_info):
+        import tempfile
+
+        from dvc.path_info import PathInfo
+        from dvc.utils import tmp_fname
+
+        # Sorting the list by path to ensure reproducibility
+        dir_info = sorted(dir_info, key=itemgetter(self.tree.PARAM_RELPATH))
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info, fobj, sort_keys=True)
+
+        from_info = PathInfo(tmp)
+        to_info = self.tree.path_info / tmp_fname("")
+        self.tree.upload(from_info, to_info, no_progress_bar=True)
+
+        hash_info = self.tree.get_file_hash(to_info)
+        hash_info.value += self.tree.CHECKSUM_DIR_SUFFIX
+        hash_info.dir_info = dir_info
+
+        return hash_info, to_info
+
+    def save_dir_info(self, dir_info, hash_info=None):
+        if (
+            hash_info
+            and hash_info.name == self.tree.PARAM_CHECKSUM
+            and not self.changed_cache_file(hash_info)
+        ):
+            return hash_info
+
+        hi, tmp_info = self._get_dir_info_hash(dir_info)
+        new_info = self.tree.hash_to_path_info(hi.value)
+        if self.changed_cache_file(hi):
+            self.tree.makedirs(new_info.parent)
+            self.tree.move(tmp_info, new_info, mode=self.CACHE_MODE)
+
+        self.tree.state.save(new_info, hi.value)
+
+        return hi
+
+    def _save_dir(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if not hash_info.dir_info:
+            hash_info.dir_info = tree.cache.get_dir_cache(hash_info)
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
+        for entry in Tqdm(
+            hi.dir_info, desc="Saving " + path_info.name, unit="file"
+        ):
+            entry_info = path_info / entry[self.tree.PARAM_RELPATH]
+            entry_hash = HashInfo(
+                self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
+            )
+            self._save_file(
+                entry_info, tree, entry_hash, save_link=False, **kwargs
+            )
+
+        if save_link:
+            self.tree.state.save_link(path_info)
+        if self.tree.exists(path_info):
+            self.tree.state.save(path_info, hi.value)
+
+        cache_info = self.tree.hash_to_path_info(hi.value)
+        self.tree.state.save(cache_info, hi.value)
+
+    def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if path_info.scheme != self.tree.scheme:
+            raise RemoteActionNotImplemented(
+                f"save {path_info.scheme} -> {self.tree.scheme}",
+                self.tree.scheme,
+            )
+
+        self._save(path_info, tree, hash_info, save_link, **kwargs)
+
+    def _save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        to_info = self.tree.hash_to_path_info(hash_info.value)
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
+
+        if tree.isdir(path_info):
+            self._save_dir(path_info, tree, hash_info, save_link, **kwargs)
+        else:
+            self._save_file(path_info, tree, hash_info, save_link, **kwargs)
+
+    # Override to return path as a string instead of PathInfo for clouds
+    # which support string paths (see local)
+    def hash_to_path(self, hash_):
+        return self.tree.hash_to_path_info(hash_)
+
+    def changed_cache_file(self, hash_info):
+        """Compare the given hash with the (corresponding) actual one.
+
+        - Use `State` as a cache for computed hashes
+            + The entries are invalidated by taking into account the following:
+                * mtime
+                * inode
+                * size
+                * hash
+
+        - Remove the file from cache if it doesn't match the actual hash
+        """
+        # Prefer string path over PathInfo when possible due to performance
+        cache_info = self.hash_to_path(hash_info.value)
+        if self.tree.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
+        actual = self.tree.get_hash(cache_info)
+
+        logger.debug(
+            "cache '%s' expected '%s' actual '%s'",
+            cache_info,
+            hash_info,
+            actual,
+        )
+
+        if not hash_info or not actual:
+            return True
+
+        if actual.value.split(".")[0] == hash_info.value.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.tree.protect(cache_info)
+            return False
+
+        if self.tree.exists(cache_info):
+            logger.warning("corrupted cache file '%s'.", cache_info)
+            self.tree.remove(cache_info)
+
+        return True
+
+    def _changed_dir_cache(self, hash_info, path_info=None, filter_info=None):
+        if self.changed_cache_file(hash_info):
+            return True
+
+        for entry in self.get_dir_cache(hash_info):
+            entry_hash = HashInfo(
+                self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
+            )
+
+            if path_info and filter_info:
+                entry_info = path_info / entry[self.tree.PARAM_RELPATH]
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
+            if self.changed_cache_file(entry_hash):
+                return True
+
+        return False
+
+    def changed_cache(self, hash_info, path_info=None, filter_info=None):
+        if hash_info.isdir:
+            return self._changed_dir_cache(
+                hash_info, path_info=path_info, filter_info=filter_info
+            )
+        return self.changed_cache_file(hash_info)
+
+    def already_cached(self, path_info):
+        _, current = self.tree.get_hash(path_info)
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.tree.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                " Are you sure you want to proceed?".format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise ConfirmRemoveError(str(path_info))
+
+        self.tree.remove(path_info)
+
+    def _checkout_file(
+        self, path_info, hash_info, force, progress_callback=None, relink=False
+    ):
+        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
+        if self.tree.exists(path_info):
+            logger.debug("data '%s' will be replaced.", path_info)
+            self.safe_remove(path_info, force=force)
+            added, modified = False, True
+
+        self.link(cache_info, path_info)
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_info.value)
+        if progress_callback:
+            progress_callback(str(path_info))
+
+        return added, modified and not relink
+
+    def _checkout_dir(
+        self,
+        path_info,
+        hash_info,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        added, modified = False, False
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.tree.exists(path_info):
+            added = True
+            self.tree.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(hash_info)
+
+        logger.debug("Linking directory '%s'.", path_info)
+
+        for entry in dir_info:
+            relative_path = entry[self.tree.PARAM_RELPATH]
+            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+            entry_cache_info = self.tree.hash_to_path_info(entry_hash)
+            entry_info = path_info / relative_path
+
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
+            entry_hash_info = HashInfo(self.tree.PARAM_CHECKSUM, entry_hash)
+            if relink or self.changed(entry_info, entry_hash_info):
+                modified = True
+                self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.tree.state.save(entry_info, entry_hash)
+            if progress_callback:
+                progress_callback(str(entry_info))
+
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )
+
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_info.value)
+
+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(self.tree.walk_files(path_info))
+
+        needed_files = {
+            path_info / entry[self.tree.PARAM_RELPATH] for entry in dir_info
+        }
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
+            self.safe_remove(path, force)
+
+        return bool(redundant_files)
+
+    def checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if path_info.scheme not in ["local", self.tree.scheme]:
+            raise NotImplementedError
+
+        failed = None
+        skip = False
+        if not hash_info:
+            logger.warning(
+                "No file hash info found for '%s'. " "It won't be created.",
+                path_info,
+            )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        elif not relink and not self.changed(path_info, hash_info):
+            logger.debug("Data '%s' didn't change.", path_info)
+            skip = True
+
+        elif self.changed_cache(
+            hash_info, path_info=path_info, filter_info=filter_info
+        ):
+            logger.warning(
+                "Cache '%s' not found. File '%s' won't be created.",
+                hash_info,
+                path_info,
+            )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        if failed or skip:
+            if progress_callback:
+                progress_callback(
+                    str(path_info),
+                    self.get_files_number(
+                        self.tree.path_info, hash_info, filter_info
+                    ),
+                )
+            if failed:
+                raise CheckoutError([failed])
+            return
+
+        logger.debug(
+            "Checking out '%s' with cache '%s'.", path_info, hash_info
+        )
+
+        return self._checkout(
+            path_info,
+            hash_info,
+            force,
+            progress_callback,
+            relink,
+            filter_info,
+        )
+
+    def _checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if not hash_info.isdir:
+            return self._checkout_file(
+                path_info, hash_info, force, progress_callback, relink
+            )
+
+        return self._checkout_dir(
+            path_info, hash_info, force, progress_callback, relink, filter_info
+        )
+
+    def get_files_number(self, path_info, hash_info, filter_info):
+        from funcy.py3 import ilen
+
+        if not hash_info:
+            return 0
+
+        if not hash_info.isdir:
+            return 1
+
+        if not filter_info:
+            return len(self.get_dir_cache(hash_info))
+
+        return ilen(
+            filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
+            for entry in self.get_dir_cache(hash_info)
+        )
+
+    def _to_dict(self, dir_info):
+        return {
+            entry[self.tree.PARAM_RELPATH]: entry[self.tree.PARAM_CHECKSUM]
+            for entry in dir_info
+        }
+
+    def _from_dict(self, dir_dict):
+        return [
+            {
+                self.tree.PARAM_RELPATH: relpath,
+                self.tree.PARAM_CHECKSUM: checksum,
+            }
+            for relpath, checksum in dir_dict.items()
+        ]
+
+    @staticmethod
+    def _diff(ancestor, other, allow_removed=False):
+        from dictdiffer import diff
+
+        allowed = ["add"]
+        if allow_removed:
+            allowed.append("remove")
+
+        result = list(diff(ancestor, other))
+        for typ, _, _ in result:
+            if typ not in allowed:
+                raise MergeError(
+                    "unable to auto-merge directories with diff that contains "
+                    f"'{typ}'ed files"
+                )
+        return result
+
+    def _merge_dirs(self, ancestor_info, our_info, their_info):
+        from dictdiffer import patch
+
+        ancestor = self._to_dict(ancestor_info)
+        our = self._to_dict(our_info)
+        their = self._to_dict(their_info)
+
+        our_diff = self._diff(ancestor, our)
+        if not our_diff:
+            return self._from_dict(their)
+
+        their_diff = self._diff(ancestor, their)
+        if not their_diff:
+            return self._from_dict(our)
+
+        # make sure there are no conflicting files
+        self._diff(our, their, allow_removed=True)
+
+        merged = patch(our_diff + their_diff, ancestor, in_place=True)
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(
+            self._from_dict(merged), key=itemgetter(self.tree.PARAM_RELPATH)
+        )
+
+    def merge(self, ancestor_info, our_info, their_info):
+        assert our_info
+        assert their_info
+
+        if ancestor_info:
+            ancestor = self.get_dir_cache(ancestor_info)
+        else:
+            ancestor = []
+
+        our = self.get_dir_cache(our_info)
+        their = self.get_dir_cache(their_info)
+
+        merged = self._merge_dirs(ancestor, our, their)
+        return self.save_dir_info(merged)
+
+    def get_hash(self, tree, path_info):
+        hash_info = tree.get_hash(path_info)
+        if not hash_info.isdir:
+            assert hash_info.name == self.tree.PARAM_CHECKSUM
+            return hash_info
+
+        return self.save_dir_info(hash_info.dir_info, hash_info)

dvc/cache/local.py

@@ -0,0 +1,476 @@
+import errno
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial, wraps
+
+from funcy import cached_property, concat
+
+from dvc.exceptions import DownloadError, UploadError
+from dvc.hash_info import HashInfo
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+
+from ..remote.base import index_locked
+from ..tree.local import LocalTree
+from .base import (
+    STATUS_DELETED,
+    STATUS_MAP,
+    STATUS_MISSING,
+    STATUS_NEW,
+    CloudCache,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _log_exceptions(func, operation):
+    @wraps(func)
+    def wrapper(from_info, to_info, *args, **kwargs):
+        try:
+            func(from_info, to_info, *args, **kwargs)
+            return 0
+        except Exception as exc:  # pylint: disable=broad-except
+            # NOTE: this means we ran out of file descriptors and there is no
+            # reason to try to proceed, as we will hit this error anyways.
+            # pylint: disable=no-member
+            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+                raise
+
+            logger.exception(
+                "failed to %s '%s' to '%s'", operation, from_info, to_info
+            )
+            return 1
+
+    return wrapper
+
+
+class LocalCache(CloudCache):
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+    CACHE_MODE = LocalTree.CACHE_MODE
+
+    def __init__(self, tree):
+        super().__init__(tree)
+        self.cache_dir = tree.config.get("url")
+
+    @property
+    def cache_dir(self):
+        return self.tree.path_info.fspath if self.tree.path_info else None
+
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.tree.path_info = PathInfo(value) if value else None
+
+    @classmethod
+    def supported(cls, config):  # pylint: disable=unused-argument
+        return True
+
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def hash_to_path(self, hash_):
+        # NOTE: `self.cache_path` is already normalized so we can simply use
+        # `os.sep` instead of `os.path.join`. This results in this helper
+        # being ~5.5 times faster.
+        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
+
+    def hashes_exist(
+        self, hashes, jobs=None, name=None
+    ):  # pylint: disable=unused-argument
+        return [
+            hash_
+            for hash_ in Tqdm(
+                hashes,
+                unit="file",
+                desc="Querying "
+                + ("cache in " + name if name else "local cache"),
+            )
+            if not self.changed_cache_file(
+                HashInfo(self.tree.PARAM_CHECKSUM, hash_)
+            )
+        ]
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        current = self.tree.get_hash(path_info)
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
+    @index_locked
+    def status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+        log_missing=True,
+    ):
+        # Return flattened dict containing all status info
+        dir_status, file_status, _ = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+            log_missing=log_missing,
+        )
+        return dict(dir_status, **file_status)
+
+    def _status(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+        log_missing=True,
+    ):
+        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
+
+        dir_status_info contains status for .dir files, file_status_info
+        contains status for all other files, and dir_contents is a dict of
+        {dir_hash: set(file_hash, ...)} which can be used to map
+        a .dir file to its file contents.
+        """
+        logger.debug(
+            f"Preparing to collect status from {remote.tree.path_info}"
+        )
+        md5s = set(named_cache.scheme_keys(self.tree.scheme))
+
+        logger.debug("Collecting information from local cache...")
+        local_exists = frozenset(
+            self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
+        )
+
+        # This is a performance optimization. We can safely assume that,
+        # if the resources that we want to fetch are already cached,
+        # there's no need to check the remote storage for the existence of
+        # those files.
+        if download and local_exists == md5s:
+            remote_exists = local_exists
+        else:
+            logger.debug("Collecting information from remote cache...")
+            remote_exists = set()
+            dir_md5s = set(named_cache.dir_keys(self.tree.scheme))
+            if dir_md5s:
+                remote_exists.update(
+                    self._indexed_dir_hashes(named_cache, remote, dir_md5s)
+                )
+                md5s.difference_update(remote_exists)
+            if md5s:
+                remote_exists.update(
+                    remote.hashes_exist(
+                        md5s, jobs=jobs, name=str(remote.tree.path_info)
+                    )
+                )
+        return self._make_status(
+            named_cache,
+            show_checksums,
+            local_exists,
+            remote_exists,
+            log_missing,
+        )
+
+    def _make_status(
+        self,
+        named_cache,
+        show_checksums,
+        local_exists,
+        remote_exists,
+        log_missing,
+    ):
+        def make_names(hash_, names):
+            return {"name": hash_ if show_checksums else " ".join(names)}
+
+        dir_status = {}
+        file_status = {}
+        dir_contents = {}
+        for hash_, item in named_cache[self.tree.scheme].items():
+            if item.children:
+                dir_status[hash_] = make_names(hash_, item.names)
+                dir_contents[hash_] = set()
+                for child_hash, child in item.children.items():
+                    file_status[child_hash] = make_names(
+                        child_hash, child.names
+                    )
+                    dir_contents[hash_].add(child_hash)
+            else:
+                file_status[hash_] = make_names(hash_, item.names)
+
+        self._fill_statuses(dir_status, local_exists, remote_exists)
+        self._fill_statuses(file_status, local_exists, remote_exists)
+
+        if log_missing:
+            self._log_missing_caches(dict(dir_status, **file_status))
+
+        return dir_status, file_status, dir_contents
+
+    def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
+        # Validate our index by verifying all indexed .dir hashes
+        # still exist on the remote
+        indexed_dirs = set(remote.index.dir_hashes())
+        indexed_dir_exists = set()
+        if indexed_dirs:
+            indexed_dir_exists.update(
+                remote.tree.list_hashes_exists(indexed_dirs)
+            )
+            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
+            if missing_dirs:
+                logger.debug(
+                    "Remote cache missing indexed .dir hashes '{}', "
+                    "clearing remote index".format(", ".join(missing_dirs))
+                )
+                remote.index.clear()
+
+        # Check if non-indexed (new) dir hashes exist on remote
+        dir_exists = dir_md5s.intersection(indexed_dir_exists)
+        dir_exists.update(
+            remote.tree.list_hashes_exists(dir_md5s - dir_exists)
+        )
+
+        # If .dir hash exists on the remote, assume directory contents
+        # still exists on the remote
+        for dir_hash in dir_exists:
+            file_hashes = list(
+                named_cache.child_keys(self.tree.scheme, dir_hash)
+            )
+            if dir_hash not in remote.index:
+                logger.debug(
+                    "Indexing new .dir '{}' with '{}' nested files".format(
+                        dir_hash, len(file_hashes)
+                    )
+                )
+                remote.index.update([dir_hash], file_hashes)
+            yield dir_hash
+            yield from file_hashes
+
+    @staticmethod
+    def _fill_statuses(hash_info_dir, local_exists, remote_exists):
+        # Using sets because they are way faster for lookups
+        local = set(local_exists)
+        remote = set(remote_exists)
+
+        for md5, info in hash_info_dir.items():
+            status = STATUS_MAP[(md5 in local, md5 in remote)]
+            info["status"] = status
+
+    def _get_plans(self, download, remote, status_info, status):
+        cache = []
+        path_infos = []
+        names = []
+        hashes = []
+        missing = []
+        for md5, info in Tqdm(
+            status_info.items(), desc="Analysing status", unit="file"
+        ):
+            if info["status"] == status:
+                cache.append(self.tree.hash_to_path_info(md5))
+                path_infos.append(remote.tree.hash_to_path_info(md5))
+                names.append(info["name"])
+                hashes.append(md5)
+            elif info["status"] == STATUS_MISSING:
+                missing.append(md5)
+
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return (from_infos, to_infos, names, hashes), missing
+
+    def _process(
+        self,
+        named_cache,
+        remote,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        logger.debug(
+            "Preparing to {} '{}'".format(
+                "download data from" if download else "upload data to",
+                remote.tree.path_info,
+            )
+        )
+
+        if download:
+            func = partial(
+                _log_exceptions(remote.tree.download, "download"),
+                dir_mode=self.tree.dir_mode,
+                file_mode=self.tree.file_mode,
+            )
+            status = STATUS_DELETED
+            desc = "Downloading"
+        else:
+            func = _log_exceptions(remote.tree.upload, "upload")
+            status = STATUS_NEW
+            desc = "Uploading"
+
+        if jobs is None:
+            jobs = remote.tree.JOBS
+
+        dir_status, file_status, dir_contents = self._status(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+
+        dir_plans, _ = self._get_plans(download, remote, dir_status, status)
+        file_plans, missing_files = self._get_plans(
+            download, remote, file_status, status
+        )
+
+        total = len(dir_plans[0]) + len(file_plans[0])
+        if total == 0:
+            return 0
+
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
+                if download:
+                    from_infos, to_infos, names, _ = (
+                        d + f for d, f in zip(dir_plans, file_plans)
+                    )
+                    fails = sum(
+                        executor.map(func, from_infos, to_infos, names)
+                    )
+                else:
+                    # for uploads, push files first, and any .dir files last
+
+                    file_futures = {}
+                    for from_info, to_info, name, hash_ in zip(*file_plans):
+                        file_futures[hash_] = executor.submit(
+                            func, from_info, to_info, name
+                        )
+                    dir_futures = {}
+                    for from_info, to_info, name, dir_hash in zip(*dir_plans):
+                        # if for some reason a file contained in this dir is
+                        # missing both locally and in the remote, we want to
+                        # push whatever file content we have, but should not
+                        # push .dir file
+                        for file_hash in missing_files:
+                            if file_hash in dir_contents[dir_hash]:
+                                logger.debug(
+                                    "directory '%s' contains missing files,"
+                                    "skipping .dir file upload",
+                                    name,
+                                )
+                                break
+                        else:
+                            wait_futures = {
+                                future
+                                for file_hash, future in file_futures.items()
+                                if file_hash in dir_contents[dir_hash]
+                            }
+                            dir_futures[dir_hash] = executor.submit(
+                                self._dir_upload,
+                                func,
+                                wait_futures,
+                                from_info,
+                                to_info,
+                                name,
+                            )
+                    fails = sum(
+                        future.result()
+                        for future in concat(
+                            file_futures.values(), dir_futures.values()
+                        )
+                    )
+
+        if fails:
+            if download:
+                remote.index.clear()
+                raise DownloadError(fails)
+            raise UploadError(fails)
+
+        if not download:
+            # index successfully pushed dirs
+            for dir_hash, future in dir_futures.items():
+                if future.result() == 0:
+                    file_hashes = dir_contents[dir_hash]
+                    logger.debug(
+                        "Indexing pushed dir '{}' with "
+                        "'{}' nested files".format(dir_hash, len(file_hashes))
+                    )
+                    remote.index.update([dir_hash], file_hashes)
+
+        return len(dir_plans[0]) + len(file_plans[0])
+
+    @staticmethod
+    def _dir_upload(func, futures, from_info, to_info, name):
+        for future in as_completed(futures):
+            if future.result():
+                # do not upload this .dir file if any file in this
+                # directory failed to upload
+                logger.debug(
+                    "failed to upload full contents of '{}', "
+                    "aborting .dir file upload".format(name)
+                )
+                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
+                return 1
+        return func(from_info, to_info, name)
+
+    @index_locked
+    def push(self, named_cache, remote, jobs=None, show_checksums=False):
+        return self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
+
+    @index_locked
+    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
+        ret = self._process(
+            named_cache,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )
+
+        if not remote.tree.verify:
+            for checksum in named_cache.scheme_keys("local"):
+                cache_file = self.tree.hash_to_path_info(checksum)
+                if self.tree.exists(cache_file):
+                    # We can safely save here, as existing corrupted files will
+                    # be removed upon status, while files corrupted during
+                    # download will not be moved from tmp_file
+                    # (see `BaseTree.download()`)
+                    self.tree.state.save(cache_file, checksum)
+
+        return ret
+
+    @staticmethod
+    def _log_missing_caches(hash_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in hash_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "\n".join(
+                "name: {}, md5: {}".format(info["name"], md5)
+                for md5, info in missing_caches
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files:\n{}".format(missing_desc)
+            )
+            logger.warning(msg)

dvc/cli.py

@@ -6,31 +6,36 @@ import sys
 from .command import (
     add,
     cache,
+    check_ignore,
     checkout,
     commit,
+    completion,
     config,
     daemon,
+    dag,
     data_sync,
     destroy,
     diff,
+    experiments,
+    freeze,
     gc,
     get,
     get_url,
+    git_hook,
     imp,
     imp_url,
     init,
     install,
     ls,
-    lock,
     metrics,
     move,
-    pipeline,
+    params,
+    plots,
     remote,
     remove,
     repro,
     root,
     run,
-    tag,
     unprotect,
     update,
     version,
@@ -38,7 +43,6 @@ from .command import (
 from .command.base import fix_subparsers
 from .exceptions import DvcParserError
 
-
 logger = logging.getLogger(__name__)
 
 COMMANDS = [
@@ -61,34 +65,56 @@ COMMANDS = [
     remote,
     cache,
     metrics,
+    params,
     install,
     root,
     ls,
-    lock,
-    pipeline,
+    freeze,
+    dag,
     daemon,
     commit,
-    tag,
+    completion,
     diff,
     version,
     update,
+    git_hook,
+    plots,
+    experiments,
+    check_ignore,
 ]
 
 
+def _find_parser(parser, cmd_cls):
+    defaults = parser._defaults  # pylint: disable=protected-access
+    if not cmd_cls or cmd_cls == defaults.get("func"):
+        parser.print_help()
+        raise DvcParserError()
+
+    actions = parser._actions  # pylint: disable=protected-access
+    for action in actions:
+        if not isinstance(action.choices, dict):
+            # NOTE: we are only interested in subparsers
+            continue
+        for subparser in action.choices.values():
+            _find_parser(subparser, cmd_cls)
+
+
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
 
-    def error(self, message):
-        """Custom error method.
-        Args:
-            message (str): error message.
-
-        Raises:
-            dvc.exceptions.DvcParser: dvc parser exception.
-        """
+    def error(self, message, cmd_cls=None):  # pylint: disable=arguments-differ
         logger.error(message)
-        self.print_help()
-        raise DvcParserError()
+        _find_parser(self, cmd_cls)
+
+    def parse_args(self, args=None, namespace=None):
+        # NOTE: overriding to provide a more granular help message.
+        # E.g. `dvc plots diff --bad-flag` would result in a `dvc plots diff`
+        # help message instead of generic `dvc` usage.
+        args, argv = self.parse_known_args(args, namespace)
+        if argv:
+            msg = "unrecognized arguments: %s"
+            self.error(msg % " ".join(argv), getattr(args, "func", None))
+        return args
 
 
 class VersionAction(argparse.Action):  # pragma: no cover
@@ -111,30 +137,26 @@ def get_parent_parser():
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
 
-    log_level_group = parent_parser.add_mutually_exclusive_group()
-    log_level_group.add_argument(
-        "-q", "--quiet", action="store_true", default=False, help="Be quiet."
-    )
-    log_level_group.add_argument(
-        "-v",
-        "--verbose",
+    parent_parser.add_argument(
+        "--cprofile",
         action="store_true",
         default=False,
-        help="Be verbose.",
+        help=argparse.SUPPRESS,
+    )
+    parent_parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
+
+    log_level_group = parent_parser.add_mutually_exclusive_group()
+    log_level_group.add_argument(
+        "-q", "--quiet", action="count", default=0, help="Be quiet."
+    )
+    log_level_group.add_argument(
+        "-v", "--verbose", action="count", default=0, help="Be verbose."
     )
 
     return parent_parser
 
 
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        dvc.exceptions.DvcParserError: raised for argument parsing errors.
-    """
+def get_main_parser():
     parent_parser = get_parent_parser()
 
     # Main parser
@@ -182,6 +204,18 @@ def parse_args(argv=None):
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
-    args = parser.parse_args(argv)
+    return parser
 
+
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
     return args

dvc/command/add.py

@@ -1,7 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link, CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
@@ -18,6 +19,7 @@ class CmdAdd(CmdBase):
                 recursive=self.args.recursive,
                 no_commit=self.args.no_commit,
                 fname=self.args.file,
+                external=self.args.external,
             )
 
         except DvcException:
@@ -50,11 +52,17 @@ def add_parser(subparsers, parent_parser):
         help="Don't put files/directories into cache.",
     )
     parser.add_argument(
-        "-f",
-        "--file",
-        help="Specify name of the DVC-file this command will generate.",
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow targets that are outside of the DVC repository.",
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add."
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
     )
+    parser.add_argument(
+        "targets", nargs="+", help="Input files/directories to add.",
+    ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,5 +1,5 @@
 import logging
-
+from abc import ABC, abstractmethod
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +27,7 @@ def append_doc_link(help_message, path):
     )
 
 
-class CmdBase(object):
+class CmdBase(ABC):
     def __init__(self, args):
         from dvc.repo import Repo
         from dvc.updater import Updater
@@ -36,23 +36,22 @@ class CmdBase(object):
         self.config = self.repo.config
         self.args = args
         hardlink_lock = self.config["core"].get("hardlink_lock", False)
-        updater = Updater(self.repo.dvc_dir, hardlink_lock=hardlink_lock)
+        updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
         updater.check()
 
     @property
     def default_targets(self):
-        """Default targets for `dvc repro` and `dvc pipeline`."""
-        from dvc.stage import Stage
+        """Default targets for `dvc repro`."""
+        from dvc.dvcfile import PIPELINE_FILE
 
-        msg = "assuming default target '{}'.".format(Stage.STAGE_FILE)
-        logger.warning(msg)
-        return [Stage.STAGE_FILE]
+        logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
+        return [PIPELINE_FILE]
 
-    # Abstract methods that have to be implemented by any inheritance class
+    @abstractmethod
     def run(self):
-        raise NotImplementedError
+        pass
 
 
 class CmdBaseNoRepo(CmdBase):
-    def __init__(self, args):
+    def __init__(self, args):  # pylint: disable=super-init-not-called
         self.args = args

dvc/command/cache.py

@@ -1,5 +1,6 @@
 import argparse
 
+from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
@@ -55,5 +56,5 @@ def add_parser(subparsers, parent_parser):
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
         "config file location.",
-    )
+    ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/check_ignore.py

@@ -0,0 +1,126 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCheckIgnore(CmdBase):
+    def __init__(self, args):
+        super().__init__(args)
+        self.ignore_filter = self.repo.tree.dvcignore
+
+    def _show_results(self, result):
+        if not result.match and not self.args.non_matching:
+            return
+
+        if self.args.details:
+            patterns = result.patterns
+            if not self.args.all:
+                patterns = patterns[-1:]
+
+            for pattern in patterns:
+                logger.info("{}\t{}".format(pattern, result.file))
+        else:
+            logger.info(result.file)
+
+    def _check_one_file(self, target):
+        result = self.ignore_filter.check_ignore(target)
+        self._show_results(result)
+        if result.match:
+            return 0
+        return 1
+
+    def _interactive_mode(self):
+        ret = 1
+        while True:
+            try:
+                target = input()
+            except (KeyboardInterrupt, EOFError):
+                break
+            if target == "":
+                break
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _normal_mode(self):
+        ret = 1
+        for target in self.args.targets:
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _check_args(self):
+        if not self.args.stdin and not self.args.targets:
+            raise DvcException("`targets` or `--stdin` needed")
+
+        if self.args.stdin and self.args.targets:
+            raise DvcException("cannot have both `targets` and `--stdin`")
+
+        if self.args.non_matching and not self.args.details:
+            raise DvcException(
+                "`--non-matching` is only valid with `--details`"
+            )
+
+        if self.args.all and not self.args.details:
+            raise DvcException("`--all` is only valid with `--details`")
+
+        if self.args.quiet and self.args.details:
+            raise DvcException("cannot use both `--details` and `--quiet`")
+
+    def run(self):
+        self._check_args()
+        if self.args.stdin:
+            return self._interactive_mode()
+        return self._normal_mode()
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = (
+        "Check whether files or directories are excluded due to `.dvcignore`."
+    )
+
+    parser = subparsers.add_parser(
+        "check-ignore",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "check-ignore"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-d",
+        "--details",
+        action="store_true",
+        default=False,
+        help="Show the exclude patterns along with each target path.",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "-n",
+        "--non-matching",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "--stdin",
+        action="store_true",
+        default=False,
+        help="Read paths from standard input instead of providing `targets`.",
+    )
+    parser.add_argument(
+        "targets", nargs="*", help="File or directory paths to check",
+    ).complete = completion.FILE
+    parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -1,18 +1,68 @@
 import argparse
+import logging
+import operator
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+import colorama
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import CheckoutError
+from dvc.utils.humanize import get_summary
+
+logger = logging.getLogger(__name__)
+
+
+def log_changes(stats):
+    colors = [
+        ("modified", colorama.Fore.YELLOW,),
+        ("added", colorama.Fore.GREEN),
+        ("deleted", colorama.Fore.RED,),
+    ]
+
+    for state, color in colors:
+        entries = stats.get(state)
+
+        if not entries:
+            continue
+
+        for entry in entries:
+            logger.info(
+                "{color}{state}{nc}{spacing}{entry}".format(
+                    color=color,
+                    state=state[0].upper(),
+                    nc=colorama.Fore.RESET,
+                    spacing="\t",
+                    entry=entry,
+                )
+            )
 
 
 class CmdCheckout(CmdBase):
     def run(self):
-        self.repo.checkout(
-            targets=self.args.targets,
-            with_deps=self.args.with_deps,
-            force=self.args.force,
-            relink=self.args.relink,
-            recursive=self.args.recursive,
-        )
+        stats, exc = None, None
+        try:
+            stats = self.repo.checkout(
+                targets=self.args.targets,
+                with_deps=self.args.with_deps,
+                force=self.args.force,
+                relink=self.args.relink,
+                recursive=self.args.recursive,
+            )
+        except CheckoutError as _exc:
+            exc = _exc
+            stats = exc.stats
+
+        if self.args.summary:
+            default_message = "No changes."
+            msg = get_summary(
+                sorted(stats.items(), key=operator.itemgetter(0))
+            )
+            logger.info(msg or default_message)
+        else:
+            log_changes(stats)
+
+        if exc:
+            raise exc
         return 0
 
 
@@ -26,6 +76,12 @@ def add_parser(subparsers, parent_parser):
         help=CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    checkout_parser.add_argument(
+        "--summary",
+        action="store_true",
+        default=False,
+        help="Show summary of the changes.",
+    )
     checkout_parser.add_argument(
         "-d",
         "--with-deps",
@@ -56,7 +112,9 @@ def add_parser(subparsers, parent_parser):
     checkout_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-files to checkout. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -68,5 +67,5 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to commit. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
-    )
+    ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -0,0 +1,57 @@
+import argparse
+import logging
+
+import shtab
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+
+logger = logging.getLogger(__name__)
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
+}
+PREAMBLE = {
+    "bash": """
+# $1=COMP_WORDS[1]
+_dvc_compgen_DVCFiles() {
+  compgen -d -S '/' -- $1  # recurse into subdirs
+  compgen -f -X '!*?.dvc' -- $1
+  compgen -f -X '!*Dvcfile' -- $1
+  compgen -f -X '!*dvc.yaml' -- $1
+}
+""",
+    "zsh": "",
+}
+
+
+class CmdCompletion(CmdBaseNoRepo):
+    def run(self):
+        from dvc.cli import get_main_parser
+
+        parser = get_main_parser()
+        shell = self.args.shell
+        script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
+        print(script)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMPLETION_HELP = "Generate shell tab completion."
+    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
+    completion_parser = subparsers.add_parser(
+        "completion",
+        parents=[parent_parser],
+        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
+        help=COMPLETION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    completion_parser.add_argument(
+        "-s",
+        "--shell",
+        help="Shell syntax for completions.",
+        default="bash",
+        choices=["bash", "zsh"],
+    )
+    completion_parser.set_defaults(func=CmdCompletion)

dvc/command/config.py

@@ -1,7 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link, CmdBaseNoRepo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.config import Config, ConfigError
 
 logger = logging.getLogger(__name__)
@@ -50,21 +50,22 @@ class CmdConfig(CmdBaseNoRepo):
 
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
-parent_config_parser.add_argument(
+level_group = parent_config_parser.add_mutually_exclusive_group()
+level_group.add_argument(
     "--global",
     dest="level",
     action="store_const",
     const="global",
     help="Use global config.",
 )
-parent_config_parser.add_argument(
+level_group.add_argument(
     "--system",
     dest="level",
     action="store_const",
     const="system",
     help="Use system config.",
 )
-parent_config_parser.add_argument(
+level_group.add_argument(
     "--local",
     dest="level",
     action="store_const",

dvc/command/daemon.py

@@ -1,5 +1,5 @@
-from dvc.command.base import CmdBaseNoRepo
-from dvc.command.base import fix_subparsers
+from dvc.command import completion
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):
@@ -9,15 +9,17 @@ class CmdDaemonBase(CmdBaseNoRepo):
 class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
-        from dvc.repo import Repo
+
         from dvc.config import Config
+        from dvc.repo import Repo
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()
         dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
+        tmp_dir = os.path.join(dvc_dir, "tmp")
         config = Config(dvc_dir, validate=False)
         hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
-        updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
+        updater = Updater(tmp_dir, hardlink_lock=hardlink_lock)
         updater.fetch(detach=False)
 
         return 0
@@ -64,5 +66,7 @@ def add_parser(subparsers, parent_parser):
         description=DAEMON_ANALYTICS_HELP,
         help=DAEMON_ANALYTICS_HELP,
     )
-    daemon_analytics_parser.add_argument("target", help="Analytics file.")
+    daemon_analytics_parser.add_argument(
+        "target", help="Analytics file.",
+    ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -0,0 +1,117 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _show_ascii(G):
+    from dvc.dagascii import draw
+    from dvc.repo.graph import get_pipelines
+
+    pipelines = get_pipelines(G)
+
+    ret = []
+    for pipeline in pipelines:
+        ret.append(draw(pipeline.nodes, pipeline.edges))
+
+    return "\n".join(ret)
+
+
+def _show_dot(G):
+    import io
+
+    from networkx.drawing.nx_pydot import write_dot
+
+    dot_file = io.StringIO()
+    write_dot(G, dot_file)
+    return dot_file.getvalue()
+
+
+def _build(G, target=None, full=False):
+    import networkx as nx
+
+    from dvc.repo.graph import get_pipeline, get_pipelines
+
+    if target:
+        H = get_pipeline(get_pipelines(G), target)
+        if not full:
+            descendants = nx.descendants(G, target)
+            descendants.add(target)
+            H.remove_nodes_from(set(G.nodes()) - descendants)
+    else:
+        H = G
+
+    def _relabel(stage):
+        return stage.addressing
+
+    return nx.relabel_nodes(H, _relabel, copy=False)
+
+
+class CmdDAG(CmdBase):
+    def run(self):
+        try:
+            target = None
+            if self.args.target:
+                stages = self.repo.collect(self.args.target)
+                if len(stages) > 1:
+                    logger.error(
+                        f"'{self.args.target}' contains more than one stage "
+                        "{stages}, please specify one stage"
+                    )
+                    return 1
+                target = stages[0]
+
+            G = _build(self.repo.graph, target=target, full=self.args.full,)
+
+            if self.args.dot:
+                logger.info(_show_dot(G))
+            else:
+                from dvc.utils.pager import pager
+
+                pager(_show_ascii(G))
+
+            return 0
+        except DvcException:
+            msg = "failed to show "
+            if self.args.target:
+                msg += f"a pipeline for '{target}'"
+            else:
+                msg += "pipelines"
+            logger.exception(msg)
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    DAG_HELP = "Visualize DVC project DAG."
+    dag_parser = subparsers.add_parser(
+        "dag",
+        parents=[parent_parser],
+        description=append_doc_link(DAG_HELP, "dag"),
+        help=DAG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    dag_parser.add_argument(
+        "--dot",
+        action="store_true",
+        default=False,
+        help="Print DAG with .dot format.",
+    )
+    dag_parser.add_argument(
+        "--full",
+        action="store_true",
+        default=False,
+        help=(
+            "Show full DAG that the target belongs too, instead of "
+            "showing DAG consisting only of ancestors."
+        ),
+    )
+    dag_parser.add_argument(
+        "target",
+        nargs="?",
+        help="Stage or output to show pipeline for (optional). "
+        "Finds all stages in the workspace by default.",
+    )
+    dag_parser.set_defaults(func=CmdDAG)

dvc/command/data_sync.py

@@ -1,38 +1,46 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.checkout import log_changes
+from dvc.exceptions import CheckoutError, DvcException
+from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
-    @classmethod
-    def check_up_to_date(cls, processed_files_count):
-        if processed_files_count == 0:
-            logger.info("Everything is up to date.")
+    def log_summary(self, stats):
+        default_msg = "Everything is up to date."
+        logger.info(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
+    def log_summary(self, stats):
+        log_changes(stats)
+        super().log_summary(stats)
+
     def run(self):
         try:
-            processed_files_count = self.repo.pull(
+            stats = self.repo.pull(
                 targets=self.args.targets,
                 jobs=self.args.jobs,
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 force=self.args.force,
                 recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
             )
-        except DvcException:
+            self.log_summary(stats)
+        except (CheckoutError, DvcException) as exc:
+            self.log_summary(getattr(exc, "stats", {}))
             logger.exception("failed to pull data from the cloud")
             return 1
-        self.check_up_to_date(processed_files_count)
+
         return 0
 
 
@@ -45,13 +53,15 @@ class CmdDataPush(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
             )
+            self.log_summary({"pushed": processed_files_count})
         except DvcException:
             logger.exception("failed to push data to the cloud")
             return 1
-        self.check_up_to_date(processed_files_count)
         return 0
 
 
@@ -64,13 +74,15 @@ class CmdDataFetch(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
+                run_cache=self.args.run_cache,
             )
+            self.log_summary({"fetched": processed_files_count})
         except DvcException:
             logger.exception("failed to fetch data from the cloud")
             return 1
-        self.check_up_to_date(processed_files_count)
         return 0
 
 
@@ -78,27 +90,37 @@ def shared_parent_parser():
     from dvc.cli import get_parent_parser
 
     # Parent parser used in pull/push/status
-    shared_parent_parser = argparse.ArgumentParser(
+    parent_parser = argparse.ArgumentParser(
         add_help=False, parents=[get_parent_parser()]
     )
-    shared_parent_parser.add_argument(
-        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
+    parent_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
     )
-    shared_parent_parser.add_argument(
+    parent_parser.add_argument(
         "targets",
         nargs="*",
-        help="Limit command scope to these DVC-files. "
-        "Using -R, directories to search DVC-files in can also be given.",
-    )
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVC_FILE
 
-    return shared_parent_parser
+    return parent_parser
 
 
 def add_parser(subparsers, _parent_parser):
     from dvc.command.status import CmdDataStatus
 
     # Pull
-    PULL_HELP = "Pull data files from a DVC remote storage."
+    PULL_HELP = "Download tracked files or directories from remote storage."
 
     pull_parser = subparsers.add_parser(
         "pull",
@@ -108,7 +130,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
-        "-r", "--remote", help="Remote repository to pull from."
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
     )
     pull_parser.add_argument(
         "-a",
@@ -124,6 +146,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for all tags.",
     )
+    pull_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all commits.",
+    )
     pull_parser.add_argument(
         "-f",
         "--force",
@@ -145,11 +173,16 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Pull cache for subdirectories of the specified directory.",
     )
+    pull_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Fetch run history for all stages.",
+    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    PUSH_HELP = "Uploads files or directories tracked by DVC"
-    " to remote storage."
+    PUSH_HELP = "Upload tracked files or directories to remote storage."
 
     push_parser = subparsers.add_parser(
         "push",
@@ -159,7 +192,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
-        "-r", "--remote", help="Remote repository to push to."
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
     )
     push_parser.add_argument(
         "-a",
@@ -175,6 +208,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Push cache for all tags.",
     )
+    push_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Push cache for all commits.",
+    )
     push_parser.add_argument(
         "-d",
         "--with-deps",
@@ -187,13 +226,21 @@ def add_parser(subparsers, _parent_parser):
         "--recursive",
         action="store_true",
         default=False,
-        help="Push cache from subdirectories of specified directory.",
+        help="Push cache for subdirectories of specified directory.",
+    )
+    push_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Push run history for all stages.",
     )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = "Get files or directories tracked by DVC from remote storage"
-    " into the cache."
+    FETCH_HELP = (
+        "Get tracked files or directories from remote storage"
+        " into the cache."
+    )
 
     fetch_parser = subparsers.add_parser(
         "fetch",
@@ -203,7 +250,10 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
-        "-r", "--remote", help="Remote repository to fetch from."
+        "-r",
+        "--remote",
+        help="Remote storage to fetch from",
+        metavar="<name>",
     )
     fetch_parser.add_argument(
         "-a",
@@ -219,6 +269,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for all tags.",
     )
+    fetch_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Fetch cache for all commits.",
+    )
     fetch_parser.add_argument(
         "-d",
         "--with-deps",
@@ -233,6 +289,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch cache for subdirectories of specified directory.",
     )
+    fetch_parser.add_argument(
+        "--run-cache",
+        action="store_true",
+        default=False,
+        help="Fetch run history for all stages.",
+    )
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
@@ -266,7 +328,10 @@ def add_parser(subparsers, _parent_parser):
         help="Show status of a local cache compared to a remote repository.",
     )
     status_parser.add_argument(
-        "-r", "--remote", help="Remote repository to compare local cache to."
+        "-r",
+        "--remote",
+        help="Remote storage to compare local cache to",
+        metavar="<name>",
     )
     status_parser.add_argument(
         "-a",
@@ -284,6 +349,13 @@ def add_parser(subparsers, _parent_parser):
         help="Show status of a local cache compared to a remote repository "
         "for all tags.",
     )
+    status_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show status of a local cache compared to a remote repository "
+        "for all commits.",
+    )
     status_parser.add_argument(
         "-d",
         "--with-deps",
@@ -291,4 +363,18 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Show status for all dependencies of the specified target.",
     )
+    status_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Show status of all stages in the specified directory.",
+    )
+    status_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show status in JSON format.",
+    )
+
     status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -2,11 +2,9 @@ import argparse
 import logging
 
 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -27,7 +25,7 @@ class CmdDestroy(CmdBase):
                 )
 
             self.repo.destroy()
-        except Exception:
+        except DvcException:
             logger.exception("failed to destroy DVC")
             return 1
         return 0

dvc/command/diff.py

@@ -8,19 +8,46 @@ import colorama
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
+def _digest(checksum):
+    if isinstance(checksum, str):
+        return checksum[0:8]
+    return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+
+def _show_md(diff, show_hash=False, hide_missing=False):
+    from dvc.utils.diff import table
+
+    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
+    rows = []
+    statuses = ["added", "deleted", "modified"]
+    if not hide_missing:
+        statuses.append("not in cache")
+    for status in statuses:
+        entries = diff.get(status, [])
+        if not entries:
+            continue
+        for entry in entries:
+            path = entry["path"]
+            if show_hash:
+                check_sum = _digest(entry.get("hash", ""))
+                rows.append([status, check_sum, path])
+            else:
+                rows.append([status, path])
+
+    return table(header, rows, True)
+
+
 class CmdDiff(CmdBase):
     @staticmethod
-    def _format(diff):
+    def _format(diff, hide_missing=False):
         """
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
 
-        A group's header is colored and its entries are sorted to enhance
-        readability, for example:
+        A group's header is colored to enhance readability, for example:
 
             Added:
                 another_file.txt
@@ -28,7 +55,7 @@ class CmdDiff(CmdBase):
                 dir/
                 dir/1
 
-        An example of a diff formatted when entries contain checksums:
+        An example of a diff formatted when entries contain hash:
 
             Added:
                 d3b07384 foo
@@ -41,21 +68,20 @@ class CmdDiff(CmdBase):
         At the bottom, include a summary with the number of files per state.
         """
 
-        def _digest(checksum):
-            if type(checksum) is str:
-                return checksum[0:8]
-            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
-
         colors = {
             "added": colorama.Fore.GREEN,
             "modified": colorama.Fore.YELLOW,
             "deleted": colorama.Fore.RED,
+            "not in cache": colorama.Fore.YELLOW,
         }
 
         summary = {}
         groups = []
 
-        for state in ["added", "deleted", "modified"]:
+        states = ["added", "deleted", "modified"]
+        if not hide_missing:
+            states.append("not in cache")
+        for state in states:
             summary[state] = 0
             entries = diff[state]
 
@@ -66,7 +92,7 @@ class CmdDiff(CmdBase):
 
             for entry in entries:
                 path = entry["path"]
-                checksum = entry.get("checksum")
+                checksum = entry.get("hash")
                 summary[state] += 1 if not path.endswith(os.sep) else 0
                 content.append(
                     "{space}{checksum}{separator}{path}".format(
@@ -86,31 +112,42 @@ class CmdDiff(CmdBase):
                 )
             )
 
-        groups.append(
+        if not sum(summary.values()):
+            return None
+
+        fmt = (
             "files summary: {added} added, {deleted} deleted,"
-            " {modified} modified".format_map(summary)
+            " {modified} modified"
         )
+        if not hide_missing:
+            fmt += ", {not in cache} not in cache"
+        groups.append(fmt.format_map(summary))
 
         return "\n\n".join(groups)
 
     def run(self):
         try:
             diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+            show_hash = self.args.show_hash
+            hide_missing = self.args.b_rev or self.args.hide_missing
+            if hide_missing:
+                del diff["not in cache"]
 
-            if not any(diff.values()):
-                return 0
-
-            if not self.args.checksums:
-                for _, entries in diff.items():
+            for key, entries in diff.items():
+                entries = sorted(entries, key=lambda entry: entry["path"])
+                if not show_hash:
                     for entry in entries:
-                        del entry["checksum"]
+                        del entry["hash"]
+                diff[key] = entries
 
             if self.args.show_json:
-                res = json.dumps(diff)
-            else:
-                res = self._format(diff)
-
-            logger.info(res)
+                logger.info(json.dumps(diff))
+            elif self.args.show_md:
+                logger.info(_show_md(diff, show_hash, hide_missing))
+            elif diff:
+                output = self._format(diff, hide_missing)
+                if output:
+                    logger.info(output)
 
         except DvcException:
             logger.exception("failed to get diff")
@@ -120,9 +157,8 @@ class CmdDiff(CmdBase):
 
 def add_parser(subparsers, parent_parser):
     DIFF_DESCRIPTION = (
-        "Show changes between commits in the DVC repository,"
-        " or between a commit and the workspace."
-        "Prints the list of paths added, modified, or deleted."
+        "Show added, modified, or deleted data between commits in the DVC"
+        " repository, or between a commit and the workspace."
     )
     diff_parser = subparsers.add_parser(
         "diff",
@@ -149,9 +185,20 @@ def add_parser(subparsers, parent_parser):
         default=False,
     )
     diff_parser.add_argument(
-        "--checksums",
+        "--show-hash",
         help="Display hash value for each entry",
         action="store_true",
         default=False,
     )
+    diff_parser.add_argument(
+        "--show-md",
+        help="Show tabulated output in the Markdown format (GFM).",
+        action="store_true",
+        default=False,
+    )
+    diff_parser.add_argument(
+        "--hide-missing",
+        help="Hide missing cache file status.",
+        action="store_true",
+    )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/experiments.py

@@ -0,0 +1,554 @@
+import argparse
+import io
+import logging
+from collections import OrderedDict
+from datetime import date
+from itertools import groupby
+from typing import Iterable, Optional
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.utils.flatten import flatten
+
+logger = logging.getLogger(__name__)
+
+
+def _filter_names(
+    names: Iterable,
+    label: str,
+    include: Optional[Iterable],
+    exclude: Optional[Iterable],
+):
+    if include and exclude:
+        intersection = set(include) & set(exclude)
+        if intersection:
+            values = ", ".join(intersection)
+            raise InvalidArgumentError(
+                f"'{values}' specified in both --include-{label} and"
+                f" --exclude-{label}"
+            )
+
+    names = [tuple(name.split(".")) for name in names]
+
+    def _filter(filters, update_func):
+        filters = [tuple(name.split(".")) for name in filters]
+        for length, groups in groupby(filters, len):
+            for group in groups:
+                matches = [name for name in names if name[:length] == group]
+                if not matches:
+                    name = ".".join(group)
+                    raise InvalidArgumentError(
+                        f"'{name}' does not match any known {label}"
+                    )
+                update_func({match: None for match in matches})
+
+    if include:
+        ret = OrderedDict()
+        _filter(include, ret.update)
+    else:
+        ret = OrderedDict({name: None for name in names})
+
+    if exclude:
+        _filter(exclude, ret.difference_update)
+
+    return [".".join(name) for name in ret]
+
+
+def _update_names(names, items):
+    for name, item in items:
+        if isinstance(item, dict):
+            item = flatten(item)
+            names.update(item.keys())
+        else:
+            names[name] = None
+
+
+def _collect_names(all_experiments, **kwargs):
+    metric_names = set()
+    param_names = set()
+
+    for _, experiments in all_experiments.items():
+        for exp in experiments.values():
+            _update_names(metric_names, exp.get("metrics", {}).items())
+            _update_names(param_names, exp.get("params", {}).items())
+
+    metric_names = _filter_names(
+        sorted(metric_names),
+        "metrics",
+        kwargs.get("include_metrics"),
+        kwargs.get("exclude_metrics"),
+    )
+    param_names = _filter_names(
+        sorted(param_names),
+        "params",
+        kwargs.get("include_params"),
+        kwargs.get("exclude_params"),
+    )
+
+    return metric_names, param_names
+
+
+def _collect_rows(
+    base_rev,
+    experiments,
+    metric_names,
+    param_names,
+    precision=DEFAULT_PRECISION,
+    no_timestamp=False,
+    sort_by=None,
+    sort_order=None,
+):
+    if sort_by:
+        if sort_by in metric_names:
+            sort_type = "metrics"
+        elif sort_by in param_names:
+            sort_type = "params"
+        else:
+            raise InvalidArgumentError(f"Unknown sort column '{sort_by}'")
+        reverse = sort_order == "desc"
+        experiments = _sort_exp(experiments, sort_by, sort_type, reverse)
+
+    for i, (rev, exp) in enumerate(experiments.items()):
+        row = []
+        style = None
+        queued = "*" if exp.get("queued", False) else ""
+
+        if rev == "baseline":
+            name = exp.get("name", base_rev)
+            row.append(f"{name}")
+            style = "bold"
+        elif i < len(experiments) - 1:
+            row.append(f"├── {queued}{rev[:7]}")
+        else:
+            row.append(f"└── {queued}{rev[:7]}")
+
+        if not no_timestamp:
+            row.append(_format_time(exp.get("timestamp")))
+
+        _extend_row(
+            row, metric_names, exp.get("metrics", {}).items(), precision
+        )
+        _extend_row(row, param_names, exp.get("params", {}).items(), precision)
+
+        yield row, style
+
+
+def _sort_exp(experiments, sort_by, typ, reverse):
+    if "baseline" in experiments:
+        ret = OrderedDict({"baseline": experiments.pop("baseline")})
+    else:
+        ret = OrderedDict()
+
+    def _sort(item):
+        _, exp = item
+        for fname, item in exp.get(typ, {}).items():
+            if isinstance(item, dict):
+                item = flatten(item)
+            else:
+                item = {fname: item}
+            if sort_by in item:
+                val = item[sort_by]
+                return (val is None, val)
+        return (True, None)
+
+    ret.update(sorted(experiments.items(), key=_sort, reverse=reverse))
+    return ret
+
+
+def _format_time(timestamp):
+    if timestamp is None:
+        return "-"
+    if timestamp.date() == date.today():
+        fmt = "%I:%M %p"
+    else:
+        fmt = "%b %d, %Y"
+    return timestamp.strftime(fmt)
+
+
+def _extend_row(row, names, items, precision):
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    if not items:
+        row.extend(["-"] * len(names))
+        return
+
+    for fname, item in items:
+        if isinstance(item, dict):
+            item = flatten(item)
+        else:
+            item = {fname: item}
+        for name in names:
+            if name in item:
+                value = item[name]
+                text = str(_round(value)) if value is not None else "-"
+                row.append(text)
+            else:
+                row.append("-")
+
+
+def _parse_list(param_list):
+    ret = []
+    for param_str in param_list:
+        # we don't care about filename prefixes for show, silently
+        # ignore it if provided to keep usage consistent with other
+        # metric/param list command options
+        _, _, param_str = param_str.rpartition(":")
+        ret.extend(param_str.split(","))
+    return ret
+
+
+def _show_experiments(all_experiments, console, **kwargs):
+    from rich.table import Table
+
+    from dvc.scm.git import Git
+
+    include_metrics = _parse_list(kwargs.pop("include_metrics", []))
+    exclude_metrics = _parse_list(kwargs.pop("exclude_metrics", []))
+    include_params = _parse_list(kwargs.pop("include_params", []))
+    exclude_params = _parse_list(kwargs.pop("exclude_params", []))
+
+    metric_names, param_names = _collect_names(
+        all_experiments,
+        include_metrics=include_metrics,
+        exclude_metrics=exclude_metrics,
+        include_params=include_params,
+        exclude_params=exclude_params,
+    )
+
+    table = Table()
+    table.add_column("Experiment", no_wrap=True)
+    if not kwargs.get("no_timestamp", False):
+        table.add_column("Created")
+    for name in metric_names:
+        table.add_column(name, justify="right", no_wrap=True)
+    for name in param_names:
+        table.add_column(name, justify="left")
+
+    for base_rev, experiments in all_experiments.items():
+        if Git.is_sha(base_rev):
+            base_rev = base_rev[:7]
+
+        for row, _, in _collect_rows(
+            base_rev, experiments, metric_names, param_names, **kwargs,
+        ):
+            table.add_row(*row)
+
+    console.print(table)
+
+
+class CmdExperimentsShow(CmdBase):
+    def run(self):
+        from rich.console import Console
+
+        from dvc.utils.pager import pager
+
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            all_experiments = self.repo.experiments.show(
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                sha_only=self.args.sha,
+            )
+
+            if self.args.no_pager:
+                console = Console()
+            else:
+                # Note: rich does not currently include a native way to force
+                # infinite width for use with a pager
+                console = Console(
+                    file=io.StringIO(), force_terminal=True, width=9999
+                )
+
+            _show_experiments(
+                all_experiments,
+                console,
+                include_metrics=self.args.include_metrics,
+                exclude_metrics=self.args.exclude_metrics,
+                include_params=self.args.include_params,
+                exclude_params=self.args.exclude_params,
+                no_timestamp=self.args.no_timestamp,
+                sort_by=self.args.sort_by,
+                sort_order=self.args.sort_order,
+            )
+
+            if not self.args.no_pager:
+                pager(console.file.getvalue())
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
+
+        return 0
+
+
+class CmdExperimentsCheckout(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        self.repo.experiments.checkout(self.args.experiment)
+
+        return 0
+
+
+def _show_diff(
+    diff, title="", markdown=False, no_path=False, old=False, precision=None
+):
+    from dvc.utils.diff import table
+
+    if precision is None:
+        precision = DEFAULT_PRECISION
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    rows = []
+    for fname, diff_ in diff.items():
+        sorted_diff = OrderedDict(sorted(diff_.items()))
+        for item, change in sorted_diff.items():
+            row = [] if no_path else [fname]
+            row.append(item)
+            if old:
+                row.append(_round(change.get("old")))
+            row.append(_round(change["new"]))
+            row.append(_round(change.get("diff", "diff not supported")))
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append(title)
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
+
+
+class CmdExperimentsDiff(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            diff = self.repo.experiments.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                diffs = [("metrics", "Metric"), ("params", "Param")]
+                for key, title in diffs:
+                    table = _show_diff(
+                        diff[key],
+                        title=title,
+                        markdown=self.args.show_md,
+                        no_path=self.args.no_path,
+                        old=self.args.old,
+                        precision=self.args.precision,
+                    )
+                    if table:
+                        logger.info(table)
+                        logger.info("")
+
+        except DvcException:
+            logger.exception("failed to show experiments diff")
+            return 1
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    EXPERIMENTS_HELP = "Commands to display and compare experiments."
+
+    experiments_parser = subparsers.add_parser(
+        "experiments",
+        parents=[parent_parser],
+        aliases=["exp"],
+        description=append_doc_link(EXPERIMENTS_HELP, "experiments"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    experiments_subparsers = experiments_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc experiments CMD --help` to display "
+        "command-specific help.",
+    )
+
+    fix_subparsers(experiments_subparsers)
+
+    EXPERIMENTS_SHOW_HELP = "Print experiments."
+    experiments_show_parser = experiments_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "experiments/show"),
+        help=EXPERIMENTS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show metrics for all branches.",
+    )
+    experiments_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show metrics for all tags.",
+    )
+    experiments_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
+    experiments_show_parser.add_argument(
+        "--no-pager",
+        action="store_true",
+        default=False,
+        help="Do not pipe output into a pager.",
+    )
+    experiments_show_parser.add_argument(
+        "--include-metrics",
+        action="append",
+        default=[],
+        help="Include the specified metrics in output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-metrics",
+        action="append",
+        default=[],
+        help="Exclude the specified metrics from output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--include-params",
+        action="append",
+        default=[],
+        help="Include the specified params in output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-params",
+        action="append",
+        default=[],
+        help="Exclude the specified params from output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-by",
+        help="Sort related experiments by the specified metric or param.",
+        metavar="<metric/param>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-order",
+        help="Sort order to use with --sort-by.",
+        choices=("asc", "desc"),
+        default="asc",
+    )
+    experiments_show_parser.add_argument(
+        "--no-timestamp",
+        action="store_true",
+        default=False,
+        help="Do not show experiment timestamps.",
+    )
+    experiments_show_parser.add_argument(
+        "--sha",
+        action="store_true",
+        default=False,
+        help="Always show git commit SHAs instead of branch/tag names.",
+    )
+    experiments_show_parser.set_defaults(func=CmdExperimentsShow)
+
+    EXPERIMENTS_CHECKOUT_HELP = "Checkout experiments."
+    experiments_checkout_parser = experiments_subparsers.add_parser(
+        "checkout",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_CHECKOUT_HELP, "experiments/checkout"
+        ),
+        help=EXPERIMENTS_CHECKOUT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_checkout_parser.add_argument(
+        "experiment", help="Checkout this experiment.",
+    )
+    experiments_checkout_parser.set_defaults(func=CmdExperimentsCheckout)
+
+    EXPERIMENTS_DIFF_HELP = (
+        "Show changes between experiments in the DVC repository."
+    )
+    experiments_diff_parser = experiments_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "experiments/diff"),
+        help=EXPERIMENTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
+    )
+    experiments_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help="New experiment to compare (defaults to the current workspace)",
+    )
+    experiments_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics/params as well.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric/param value.",
+    )
+    experiments_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric/param path.",
+    )
+    experiments_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)

dvc/command/freeze.py

@@ -0,0 +1,58 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdFreezeBase(CmdBase):
+    def _run(self, func, name):
+        ret = 0
+        for target in self.args.targets:
+            try:
+                func(target)
+            except DvcException:
+                logger.exception(f"failed to {name} '{target}'")
+                ret = 1
+        return ret
+
+
+class CmdFreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.freeze, "freeze")
+
+
+class CmdUnfreeze(CmdFreezeBase):
+    def run(self):
+        return self._run(self.repo.unfreeze, "unfreeze")
+
+
+def add_parser(subparsers, parent_parser):
+    FREEZE_HELP = "Freeze stages or .dvc files."
+    freeze_parser = subparsers.add_parser(
+        "freeze",
+        parents=[parent_parser],
+        description=append_doc_link(FREEZE_HELP, "freeze"),
+        help=FREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    freeze_parser.add_argument(
+        "targets", nargs="+", help="Stages or .dvc files to freeze",
+    ).complete = completion.DVC_FILE
+    freeze_parser.set_defaults(func=CmdFreeze)
+
+    UNFREEZE_HELP = "Unfreeze stages or .dvc files."
+    unfreeze_parser = subparsers.add_parser(
+        "unfreeze",
+        parents=[parent_parser],
+        description=append_doc_link(UNFREEZE_HELP, "unfreeze"),
+        help=UNFREEZE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    unfreeze_parser.add_argument(
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+    ).complete = completion.DVC_FILE
+    unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -3,18 +3,25 @@ import logging
 import os
 
 import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-
+from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)
 
 
 class CmdGC(CmdBase):
     def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
         msg = "This will remove all cache except items used in "
 
-        msg += "the working tree"
+        msg += "the workspace"
         if self.args.all_commits:
             msg += " and all git commits"
         elif self.args.all_branches and self.args.all_tags:
@@ -47,6 +54,7 @@ class CmdGC(CmdBase):
             force=self.args.force,
             jobs=self.args.jobs,
             repos=self.args.repos,
+            workspace=self.args.workspace,
         )
         return 0
 
@@ -64,25 +72,32 @@ def add_parser(subparsers, parent_parser):
         help=GC_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep data files used in the current workspace.",
+    )
     gc_parser.add_argument(
         "-a",
         "--all-branches",
         action="store_true",
         default=False,
-        help="Keep data files for the tips of all git branches.",
+        help="Keep data files for the tips of all Git branches.",
     )
     gc_parser.add_argument(
         "-T",
         "--all-tags",
         action="store_true",
         default=False,
-        help="Keep data files for all git tags.",
+        help="Keep data files for all Git tags.",
     )
     gc_parser.add_argument(
         "--all-commits",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help="Keep data files for all Git commits.",
     )
     gc_parser.add_argument(
         "-c",
@@ -92,7 +107,10 @@ def add_parser(subparsers, parent_parser):
         help="Collect garbage in remote repository.",
     )
     gc_parser.add_argument(
-        "-r", "--remote", help="Remote storage to collect garbage in."
+        "-r",
+        "--remote",
+        help="Remote storage to collect garbage in",
+        metavar="<name>",
     )
     gc_parser.add_argument(
         "-f",
@@ -102,7 +120,15 @@ def add_parser(subparsers, parent_parser):
         help="Force garbage collection - automatically agree to all prompts.",
     )
     gc_parser.add_argument(
-        "-j", "--jobs", type=int, help="Number of jobs to run simultaneously."
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
     )
     gc_parser.add_argument(
         "-p",
@@ -113,5 +139,6 @@ def add_parser(subparsers, parent_parser):
         help="Keep data files required by these projects "
         "in addition to the current one. "
         "Useful if you share a single cache across repos.",
+        metavar="<paths>",
     )
     gc_parser.set_defaults(func=CmdGC)

dvc/command/get.py

@@ -1,10 +1,10 @@
 import argparse
 import logging
 
-from .base import append_doc_link
-from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
+from . import completion
+from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -63,13 +63,20 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     get_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to download files to"
-    )
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
+    ).complete = completion.DIR
     get_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
     )
     get_parser.add_argument(
         "--show-url",

dvc/command/get_url.py

@@ -1,10 +1,10 @@
 import argparse
 import logging
 
-from .base import append_doc_link
-from .base import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
+from . import completion
+from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -17,7 +17,7 @@ class CmdGetUrl(CmdBaseNoRepo):
             Repo.get_url(self.args.url, out=self.args.out)
             return 0
         except DvcException:
-            logger.exception("failed to get '{}'".format(self.args.url))
+            logger.exception(f"failed to get '{self.args.url}'")
             return 1
 
 
@@ -34,6 +34,6 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to."
-    )
+        "out", nargs="?", help="Destination path to put data to.",
+    ).complete = completion.DIR
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -0,0 +1,159 @@
+import logging
+import os
+
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
+from dvc.exceptions import NotDvcRepoError
+
+logger = logging.getLogger(__name__)
+
+
+class CmdHookBase(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            repo = Repo()
+            repo.close()
+        except NotDvcRepoError:
+            return 0
+
+        return self._run()
+
+    def _run(self):
+        raise NotImplementedError
+
+
+class CmdPreCommit(CmdHookBase):
+    def _run(self):
+        from dvc.main import main
+
+        return main(["status"])
+
+
+class CmdPostCheckout(CmdHookBase):
+    def _run(self):
+        # when we are running from pre-commit tool, it doesn't provide CLI
+        # flags, but instead provides respective env vars that we could use.
+        flag = os.environ.get("PRE_COMMIT_CHECKOUT_TYPE")
+        if flag is None and len(self.args.args) >= 3:
+            # see https://git-scm.com/docs/githooks#_post_checkout
+            flag = self.args.args[2]
+
+        # checking out some reference and not specific file.
+        if flag != "1":
+            return 0
+
+        # make sure we are not in the middle of a rebase/merge, so we
+        # don't accidentally break it with an unsuccessful checkout.
+        # Note that git hooks are always running in repo root.
+        if os.path.isdir(os.path.join(".git", "rebase-merge")):
+            return 0
+
+        from dvc.main import main
+
+        return main(["checkout"])
+
+
+class CmdPrePush(CmdHookBase):
+    def _run(self):
+        from dvc.main import main
+
+        return main(["push"])
+
+
+class CmdMergeDriver(CmdHookBase):
+    def _run(self):
+        from dvc.dvcfile import Dvcfile
+        from dvc.repo import Repo
+
+        dvc = Repo()
+
+        try:
+            with dvc.state:
+                ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
+                our = Dvcfile(dvc, self.args.our, verify=False)
+                their = Dvcfile(dvc, self.args.their, verify=False)
+
+                our.merge(ancestor, their)
+
+                return 0
+        finally:
+            dvc.close()
+
+
+def add_parser(subparsers, parent_parser):
+    GIT_HOOK_HELP = "Run GIT hook."
+
+    git_hook_parser = subparsers.add_parser(
+        "git-hook",
+        parents=[parent_parser],
+        description=GIT_HOOK_HELP,
+        add_help=False,
+    )
+
+    git_hook_subparsers = git_hook_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc daemon CMD --help` for command-specific help.",
+    )
+
+    fix_subparsers(git_hook_subparsers)
+
+    PRE_COMMIT_HELP = "Run pre-commit GIT hook."
+    pre_commit_parser = git_hook_subparsers.add_parser(
+        "pre-commit",
+        parents=[parent_parser],
+        description=PRE_COMMIT_HELP,
+        help=PRE_COMMIT_HELP,
+    )
+    pre_commit_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_commit_parser.set_defaults(func=CmdPreCommit)
+
+    POST_CHECKOUT_HELP = "Run post-checkout GIT hook."
+    post_checkout_parser = git_hook_subparsers.add_parser(
+        "post-checkout",
+        parents=[parent_parser],
+        description=POST_CHECKOUT_HELP,
+        help=POST_CHECKOUT_HELP,
+    )
+    post_checkout_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    post_checkout_parser.set_defaults(func=CmdPostCheckout)
+
+    PRE_PUSH_HELP = "Run pre-push GIT hook."
+    pre_push_parser = git_hook_subparsers.add_parser(
+        "pre-push",
+        parents=[parent_parser],
+        description=PRE_PUSH_HELP,
+        help=PRE_PUSH_HELP,
+    )
+    pre_push_parser.add_argument(
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+    )
+    pre_push_parser.set_defaults(func=CmdPrePush)
+
+    MERGE_DRIVER_HELP = "Run GIT merge driver."
+    merge_driver_parser = git_hook_subparsers.add_parser(
+        "merge-driver",
+        parents=[parent_parser],
+        description=MERGE_DRIVER_HELP,
+        help=MERGE_DRIVER_HELP,
+    )
+    merge_driver_parser.add_argument(
+        "--ancestor",
+        required=True,
+        help="Ancestor's version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--our",
+        required=True,
+        help="Current version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--their",
+        required=True,
+        help="Other branch's version of the conflicting file.",
+    )
+    merge_driver_parser.set_defaults(func=CmdMergeDriver)

dvc/command/imp.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -45,12 +44,19 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
-    )
+        "path", help="Path to a file or directory within the repository",
+    ).complete = completion.FILE
     import_parser.add_argument(
-        "-o", "--out", nargs="?", help="Destination path to download files to"
-    )
+        "-o",
+        "--out",
+        nargs="?",
+        help="Destination path to download files to",
+        metavar="<path>",
+    ).complete = completion.DIR
     import_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
     )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -13,7 +12,10 @@ class CmdImportUrl(CmdBase):
     def run(self):
         try:
             self.repo.imp_url(
-                self.args.url, out=self.args.out, fname=self.args.file
+                self.args.url,
+                out=self.args.out,
+                fname=self.args.file,
+                no_exec=self.args.no_exec,
             )
         except DvcException:
             logger.exception(
@@ -40,25 +42,29 @@ def add_parser(subparsers, parent_parser):
     )
     import_parser.add_argument(
         "url",
-        help="Supported urls:\n"
-        "/path/to/file\n"
-        "/path/to/directory\n"
-        "C:\\\\path\\to\\file\n"
-        "C:\\\\path\\to\\directory\n"
+        help="Location of the data to download. Supported URLs:\n"
+        "/absolute/path/to/file/or/dir\n"
+        "relative/path/to/file/or/dir\n"
+        "C:\\\\path\\to\\file\\or\\dir\n"
         "https://example.com/path/to/file\n"
-        "s3://bucket/path/to/file\n"
-        "s3://bucket/path/to/directory\n"
-        "gs://bucket/path/to/file\n"
-        "gs://bucket/path/to/directory\n"
+        "s3://bucket/key/path\n"
+        "gs://bucket/path/to/file/or/dir\n"
         "hdfs://example.com/path/to/file\n"
-        "ssh://example.com:/path/to/file\n"
-        "ssh://example.com:/path/to/directory\n"
-        "remote://myremote/path/to/file (see `dvc remote`)",
+        "ssh://example.com/absolute/path/to/file/or/dir\n"
+        "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to."
-    )
+        "out", nargs="?", help="Destination path to put files to.",
+    ).complete = completion.DIR
     import_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
+    ).complete = completion.DIR
+    import_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create DVC-file without actually downloading it.",
     )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/init.py

@@ -1,21 +1,22 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
 
 class CmdInit(CmdBaseNoRepo):
     def run(self):
-        from dvc.repo import Repo
         from dvc.exceptions import InitError
+        from dvc.repo import Repo
 
         try:
             self.repo = Repo.init(
-                ".", no_scm=self.args.no_scm, force=self.args.force
+                ".",
+                no_scm=self.args.no_scm,
+                force=self.args.force,
+                subdir=self.args.subdir,
             )
             self.config = self.repo.config
         except InitError:
@@ -44,7 +45,7 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
         help="Initiate DVC in directory that is "
-        "not tracked by any scm tool (e.g. git).",
+        "not tracked by any SCM tool (e.g. Git).",
     )
     init_parser.add_argument(
         "-f",
@@ -56,4 +57,13 @@ def add_parser(subparsers, parent_parser):
             "This operation removes local cache."
         ),
     )
+    init_parser.add_argument(
+        "--subdir",
+        action="store_true",
+        default=False,
+        help=(
+            "Necessary for running this command inside a subdirectory of a "
+            "parent SCM repository."
+        ),
+    )
     init_parser.set_defaults(func=CmdInit)

dvc/command/install.py

@@ -1,9 +1,8 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
@@ -11,8 +10,8 @@ logger = logging.getLogger(__name__)
 class CmdInstall(CmdBase):
     def run(self):
         try:
-            self.repo.install()
-        except Exception:
+            self.repo.install(self.args.use_pre_commit_tool)
+        except DvcException:
             logger.exception("failed to install DVC Git hooks")
             return 1
         return 0
@@ -27,4 +26,11 @@ def add_parser(subparsers, parent_parser):
         help=INSTALL_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    install_parser.add_argument(
+        "--use-pre-commit-tool",
+        action="store_true",
+        default=False,
+        help="Install DVC hooks using pre-commit "
+        "(https://pre-commit.com) if it is installed.",
+    )
     install_parser.set_defaults(func=CmdInstall)

dvc/command/lock.py

@@ -1,61 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.exceptions import DvcException
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdLockBase(CmdBase):
-    def _run(self, unlock):
-        for target in self.args.targets:
-            try:
-                self.repo.lock_stage(target, unlock=unlock)
-            except DvcException:
-                logger.exception(
-                    "failed to {}lock '{}'".format(
-                        "un" if unlock else "", target
-                    )
-                )
-
-                return 1
-        return 0
-
-
-class CmdLock(CmdLockBase):
-    def run(self):
-        return self._run(False)
-
-
-class CmdUnlock(CmdLockBase):
-    def run(self):
-        return self._run(True)
-
-
-def add_parser(subparsers, parent_parser):
-    LOCK_HELP = "Lock DVC-files."
-    lock_parser = subparsers.add_parser(
-        "lock",
-        parents=[parent_parser],
-        description=append_doc_link(LOCK_HELP, "lock"),
-        help=LOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    lock_parser.add_argument("targets", nargs="+", help="DVC-files to lock.")
-    lock_parser.set_defaults(func=CmdLock)
-
-    UNLOCK_HELP = "Unlock DVC-files."
-    unlock_parser = subparsers.add_parser(
-        "unlock",
-        parents=[parent_parser],
-        description=append_doc_link(UNLOCK_HELP, "unlock"),
-        help=UNLOCK_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    unlock_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to unlock."
-    )
-    unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/ls.py

@@ -1,68 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
-from dvc.exceptions import DvcException
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdList(CmdBaseNoRepo):
-    def run(self):
-        from dvc.repo import Repo
-
-        try:
-            nodes = Repo.ls(
-                self.args.url,
-                self.args.target,
-                rev=self.args.rev,
-                recursive=self.args.recursive,
-                outs_only=self.args.outs_only,
-            )
-            if nodes:
-                logger.info("\n".join(nodes))
-            return 0
-        except DvcException:
-            logger.exception("failed to list '{}'".format(self.args.url))
-            return 1
-
-
-def add_parser(subparsers, parent_parser):
-    LIST_HELP = "List files and DVC outputs in the repo."
-    list_parser = subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(LIST_HELP, "list"),
-        help=LIST_HELP,
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-    list_parser.add_argument(
-        "url",
-        help="Supported urls:\n"
-        "/path/to/file\n"
-        "/path/to/directory\n"
-        "C:\\\\path\\to\\file\n"
-        "C:\\\\path\\to\\directory\n"
-        "https://github.com/path/to/repo\n"
-        "[email protected]:path/to/repo.git\n",
-    )
-    list_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        help="Recursively list files.",
-    )
-    list_parser.add_argument(
-        "--outs-only", action="store_true", help="Show only DVC outputs."
-    )
-    list_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. branch, tag, SHA)"
-    )
-    list_parser.add_argument(
-        "target",
-        nargs="?",
-        help="Path to directory within the repository to list outputs for",
-    )
-    list_parser.set_defaults(func=CmdList)

dvc/command/ls/__init__.py

@@ -0,0 +1,79 @@
+import argparse
+import logging
+import sys
+
+from dvc.command import completion
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.ls.ls_colors import LsColors
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _prettify(entries, with_color=False):
+    if with_color:
+        ls_colors = LsColors()
+        fmt = ls_colors.format
+    else:
+
+        def fmt(entry):
+            return entry["path"]
+
+    return [fmt(entry) for entry in entries]
+
+
+class CmdList(CmdBaseNoRepo):
+    def run(self):
+        from dvc.repo import Repo
+
+        try:
+            entries = Repo.ls(
+                self.args.url,
+                self.args.path,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+                dvc_only=self.args.dvc_only,
+            )
+            if entries:
+                entries = _prettify(entries, sys.stdout.isatty())
+                logger.info("\n".join(entries))
+            return 0
+        except DvcException:
+            logger.exception(f"failed to list '{self.args.url}'")
+            return 1
+
+
+def add_parser(subparsers, parent_parser):
+    LIST_HELP = (
+        "List repository contents, including files"
+        " and directories tracked by DVC and by Git."
+    )
+    list_parser = subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(LIST_HELP, "list"),
+        help=LIST_HELP,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    list_parser.add_argument("url", help="Location of DVC repository to list")
+    list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        help="Recursively list files.",
+    )
+    list_parser.add_argument(
+        "--dvc-only", action="store_true", help="Show only DVC outputs."
+    )
+    list_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    list_parser.add_argument(
+        "path",
+        nargs="?",
+        help="Path to directory within the repository to list outputs for",
+    ).complete = completion.DIR
+    list_parser.set_defaults(func=CmdList)

dvc/command/ls/ls_colors.py

@@ -0,0 +1,48 @@
+import os
+
+
+class LsColors:
+    default = "rs=0:di=01;34:ex=01;32"
+
+    def __init__(self, lscolors=None):
+        self._extensions = {}
+        self._codes = {}
+        self._load(lscolors or os.environ.get("LS_COLORS") or LsColors.default)
+
+    def _load(self, lscolors):
+        for item in lscolors.split(":"):
+            try:
+                code, color = item.split("=", 1)
+            except ValueError:
+                continue
+            if code.startswith("*."):
+                self._extensions[code[1:]] = color
+            else:
+                self._codes[code] = color
+
+    def format(self, entry):
+        text = entry["path"]
+
+        if entry.get("isout", False) and "out" in self._codes:
+            return self._format(text, code="out")
+
+        if entry.get("isdir", False):
+            return self._format(text, code="di")
+
+        if entry.get("isexec", False):
+            return self._format(text, code="ex")
+
+        _, ext = os.path.splitext(text)
+        return self._format(text, ext=ext)
+
+    def _format(self, text, code=None, ext=None):
+        val = None
+        if ext:
+            val = self._extensions.get(ext, None)
+        if code:
+            val = self._codes.get(code, None)
+
+        if not val:
+            return text
+        rs = self._codes.get("rs", 0)
+        return f"\033[{val}m{text}\033[{rs}m"

dvc/command/metrics.py

@@ -1,46 +1,38 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import BadMetricError
-from dvc.exceptions import DvcException
-
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import BadMetricError, DvcException
 
 logger = logging.getLogger(__name__)
 
 
-def show_metrics(metrics, all_branches=False, all_tags=False):
-    """
-    Args:
-        metrics (list): Where each element is either a `list`
-            if an xpath was specified, otherwise a `str`
-    """
+DEFAULT_PRECISION = 5
+
+
+def show_metrics(
+    metrics, all_branches=False, all_tags=False, all_commits=False
+):
+    from dvc.utils.diff import format_dict
+    from dvc.utils.flatten import flatten
+
     # When `metrics` contains a `None` key, it means that some files
     # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
     missing = metrics.pop(None, None)
 
     for branch, val in metrics.items():
-        if all_branches or all_tags:
-            logger.info("{branch}:".format(branch=branch))
+        if all_branches or all_tags or all_commits:
+            logger.info(f"{branch}:")
 
         for fname, metric in val.items():
-            if isinstance(metric, dict):
-                lines = list(metric.values())
-            elif isinstance(metric, list):
-                lines = metric
-            else:
-                lines = metric.splitlines()
+            if not isinstance(metric, dict):
+                logger.info("\t{}: {}".format(fname, str(metric)))
+                continue
 
-            if len(lines) > 1:
-                logger.info("\t{fname}:".format(fname=fname))
-
-                for line in lines:
-                    logger.info("\t\t{content}".format(content=line))
-
-            else:
-                logger.info("\t{}: {}".format(fname, metric))
+            logger.info(f"\t{fname}:")
+            for key, value in flatten(format_dict(metric)).items():
+                logger.info(f"\t\t{key}: {value}")
 
     if missing:
         raise BadMetricError(missing)
@@ -51,14 +43,23 @@ class CmdMetricsShow(CmdBase):
         try:
             metrics = self.repo.metrics.show(
                 self.args.targets,
-                typ=self.args.type,
-                xpath=self.args.xpath,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 recursive=self.args.recursive,
             )
 
-            show_metrics(metrics, self.args.all_branches, self.args.all_tags)
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(metrics))
+            else:
+                show_metrics(
+                    metrics,
+                    self.args.all_branches,
+                    self.args.all_tags,
+                    self.args.all_commits,
+                )
         except DvcException:
             logger.exception("failed to show metrics")
             return 1
@@ -66,70 +67,41 @@ class CmdMetricsShow(CmdBase):
         return 0
 
 
-class CmdMetricsModify(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.modify(
-                self.args.path, typ=self.args.type, xpath=self.args.xpath
-            )
-        except DvcException:
-            logger.exception("failed to modify metric file settings")
-            return 1
+def _show_diff(diff, markdown=False, no_path=False, old=False, precision=None):
+    from collections import OrderedDict
 
-        return 0
+    from dvc.utils.diff import table
 
+    if precision is None:
+        precision = DEFAULT_PRECISION
 
-class CmdMetricsAdd(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.add(
-                self.args.path, self.args.type, self.args.xpath
-            )
-        except DvcException:
-            msg = "failed to add metric file '{}'".format(self.args.path)
-            logger.exception(msg)
-            return 1
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
 
-        return 0
+        return val
 
-
-class CmdMetricsRemove(CmdBase):
-    def run(self):
-        try:
-            self.repo.metrics.remove(self.args.path)
-        except DvcException:
-            msg = "failed to remove metric file '{}'".format(self.args.path)
-            logger.exception(msg)
-            return 1
-
-        return 0
-
-
-def _show_diff(diff):
-    from texttable import Texttable
-
-    if not diff:
-        return "No changes."
-
-    table = Texttable()
-
-    # remove borders to make it easier for users to copy stuff
-    table.set_chars(("", "", "", ""))
-    table.set_deco(0)
-
-    rows = [["Path", "Metric", "Value", "Change"]]
+    rows = []
     for fname, mdiff in diff.items():
-        for metric, change in mdiff.items():
-            rows.append(
-                [
-                    fname,
-                    metric,
-                    change["new"],
-                    change.get("diff", "diff not supported"),
-                ]
-            )
-    table.add_rows(rows)
-    return table.draw()
+        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
+        for metric, change in sorted_mdiff.items():
+            row = [] if no_path else [fname]
+            row.append(metric)
+            if old:
+                row.append(_round(change.get("old")))
+            row.append(_round(change["new"]))
+            row.append(_round(change.get("diff", "diff not supported")))
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append("Metric")
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
 
 
 class CmdMetricsDiff(CmdBase):
@@ -139,9 +111,8 @@ class CmdMetricsDiff(CmdBase):
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
                 targets=self.args.targets,
-                typ=self.args.type,
-                xpath=self.args.xpath,
                 recursive=self.args.recursive,
+                all=self.args.all,
             )
 
             if self.args.show_json:
@@ -149,7 +120,15 @@ class CmdMetricsDiff(CmdBase):
 
                 logger.info(json.dumps(diff))
             else:
-                logger.info(_show_diff(diff))
+                table = _show_diff(
+                    diff,
+                    self.args.show_md,
+                    self.args.no_path,
+                    self.args.old,
+                    precision=self.args.precision,
+                )
+                if table:
+                    logger.info(table)
 
         except DvcException:
             logger.exception("failed to show metrics diff")
@@ -159,7 +138,7 @@ class CmdMetricsDiff(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = "Commands to add, manage, collect and display metrics."
+    METRICS_HELP = "Commands to display and compare metrics."
 
     metrics_parser = subparsers.add_parser(
         "metrics",
@@ -176,7 +155,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(metrics_subparsers)
 
-    METRICS_SHOW_HELP = "Output metric values."
+    METRICS_SHOW_HELP = "Print metrics, with optional formatting."
     metrics_show_parser = metrics_subparsers.add_parser(
         "show",
         parents=[parent_parser],
@@ -187,21 +166,11 @@ def add_parser(subparsers, parent_parser):
     metrics_show_parser.add_argument(
         "targets",
         nargs="*",
-        help="Metric files or directories (see -R) to show "
-        "(leave empty to display all)",
-    )
-    metrics_show_parser.add_argument(
-        "-t",
-        "--type",
         help=(
-            "Type of metrics (json/tsv/htsv/csv/hcsv). "
-            "It can be detected by the file extension automatically. "
-            "Unsupported types will be treated as raw."
+            "Limit command scope to these metric files. Using -R, "
+            "directories to search metric files in can also be given."
         ),
-    )
-    metrics_show_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
+    ).complete = completion.FILE
     metrics_show_parser.add_argument(
         "-a",
         "--all-branches",
@@ -216,6 +185,18 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show metrics for all tags.",
     )
+    metrics_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
+    metrics_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -228,53 +209,10 @@ def add_parser(subparsers, parent_parser):
     )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
-    METRICS_ADD_HELP = "Tag file as a metric file."
-    metrics_add_parser = metrics_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_ADD_HELP, "metrics/add"),
-        help=METRICS_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
+    METRICS_DIFF_HELP = (
+        "Show changes in metrics between commits in the DVC repository, or "
+        "between a commit and the workspace."
     )
-    metrics_add_parser.add_argument(
-        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
-    )
-    metrics_add_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
-    metrics_add_parser.add_argument("path", help="Path to a metric file.")
-    metrics_add_parser.set_defaults(func=CmdMetricsAdd)
-
-    METRICS_MODIFY_HELP = "Modify metric file options."
-    metrics_modify_parser = metrics_subparsers.add_parser(
-        "modify",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_MODIFY_HELP, "metrics/modify"),
-        help=METRICS_MODIFY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_modify_parser.add_argument(
-        "-t", "--type", help="Type of metrics (raw/json/tsv/htsv/csv/hcsv)."
-    )
-    metrics_modify_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
-    metrics_modify_parser.add_argument("path", help="Path to a metric file.")
-    metrics_modify_parser.set_defaults(func=CmdMetricsModify)
-
-    METRICS_REMOVE_HELP = "Remove files's metric tag."
-    metrics_remove_parser = metrics_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(METRICS_REMOVE_HELP, "metrics/remove"),
-        help=METRICS_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    metrics_remove_parser.add_argument("path", help="Path to a metric file.")
-    metrics_remove_parser.set_defaults(func=CmdMetricsRemove)
-
-    METRICS_DIFF_HELP = "Show changes in metrics between commits"
-    " in the DVC repository, or between a commit and the workspace."
     metrics_diff_parser = metrics_subparsers.add_parser(
         "diff",
         parents=[parent_parser],
@@ -288,28 +226,17 @@ def add_parser(subparsers, parent_parser):
     metrics_diff_parser.add_argument(
         "b_rev",
         nargs="?",
-        help=("New Git commit to compare (defaults to the current workspace)"),
+        help="New Git commit to compare (defaults to the current workspace)",
     )
     metrics_diff_parser.add_argument(
         "--targets",
         nargs="*",
         help=(
-            "Metric files or directories (see -R) to show diff for. "
-            "Shows diff for all metric files by default."
+            "Limit command scope to these metric files. Using -R, "
+            "directories to search metric files in can also be given."
         ),
-    )
-    metrics_diff_parser.add_argument(
-        "-t",
-        "--type",
-        help=(
-            "Type of metrics (json/tsv/htsv/csv/hcsv). "
-            "It can be detected by the file extension automatically. "
-            "Unsupported types will be treated as raw."
-        ),
-    )
-    metrics_diff_parser.add_argument(
-        "-x", "--xpath", help="json/tsv/htsv/csv/hcsv path."
-    )
+        metavar="<paths>",
+    ).complete = completion.FILE
     metrics_diff_parser.add_argument(
         "-R",
         "--recursive",
@@ -320,10 +247,43 @@ def add_parser(subparsers, parent_parser):
             "metric files."
         ),
     )
+    metrics_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics as well.",
+    )
     metrics_diff_parser.add_argument(
         "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
+    metrics_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    metrics_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric value.",
+    )
+    metrics_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric path.",
+    )
+    metrics_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics to `n` digits precision after the decimal point. "
+            f"Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
     metrics_diff_parser.set_defaults(func=CmdMetricsDiff)

dvc/command/move.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -38,7 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory."
-    )
-    move_parser.add_argument("dst", help="Destination path.")
+        "src", help="Source path to a data file or directory.",
+    ).complete = completion.FILE
+    move_parser.add_argument(
+        "dst", help="Destination path.",
+    ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/params.py

@@ -0,0 +1,118 @@
+import argparse
+import logging
+from collections import OrderedDict
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+def _show_diff(diff, markdown=False, no_path=False):
+    from dvc.utils.diff import table
+
+    rows = []
+    for fname, pdiff in diff.items():
+        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
+        for param, change in sorted_pdiff.items():
+            row = [] if no_path else [fname]
+            row.append(param)
+            row.append(change["old"])
+            row.append(change["new"])
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append("Param")
+    header.append("Old")
+    header.append("New")
+
+    return table(header, rows, markdown)
+
+
+class CmdParamsDiff(CmdBase):
+    def run(self):
+        try:
+            diff = self.repo.params.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                table = _show_diff(diff, self.args.show_md, self.args.no_path)
+                if table:
+                    logger.info(table)
+
+        except DvcException:
+            logger.exception("failed to show params diff")
+            return 1
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PARAMS_HELP = "Commands to display params."
+
+    params_parser = subparsers.add_parser(
+        "params",
+        parents=[parent_parser],
+        description=append_doc_link(PARAMS_HELP, "params"),
+        help=PARAMS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    params_subparsers = params_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc params CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(params_subparsers)
+
+    PARAMS_DIFF_HELP = (
+        "Show changes in params between commits in the DVC repository, or "
+        "between a commit and the workspace."
+    )
+    params_diff_parser = params_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(PARAMS_DIFF_HELP, "params/diff"),
+        help=PARAMS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    params_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
+    )
+    params_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help=("New Git commit to compare (defaults to the current workspace)"),
+    )
+    params_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged params as well.",
+    )
+    params_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    params_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    params_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show params path.",
+    )
+    params_diff_parser.set_defaults(func=CmdParamsDiff)

dvc/command/pipeline.py

@@ -1,247 +0,0 @@
-import argparse
-import logging
-
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.exceptions import DvcException
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdPipelineShow(CmdBase):
-    def _show(self, target, commands, outs, locked):
-        import networkx
-        from dvc.stage import Stage
-
-        stage = Stage.load(self.repo, target)
-        G = self.repo.graph
-        stages = networkx.dfs_postorder_nodes(G, stage)
-
-        if locked:
-            stages = [s for s in stages if s.locked]
-
-        for stage in stages:
-            if commands:
-                if stage.cmd is None:
-                    continue
-                logger.info(stage.cmd)
-            elif outs:
-                for out in stage.outs:
-                    logger.info(str(out))
-            else:
-                logger.info(stage.path_in_repo)
-
-    def _build_graph(self, target, commands, outs):
-        import networkx
-        from dvc.stage import Stage
-        from dvc.repo.graph import get_pipeline
-
-        target_stage = Stage.load(self.repo, target)
-        G = get_pipeline(self.repo.pipelines, target_stage)
-
-        nodes = set()
-        for stage in networkx.dfs_preorder_nodes(G, target_stage):
-            if commands:
-                if stage.cmd is None:
-                    continue
-                nodes.add(stage.cmd)
-            elif outs:
-                for out in stage.outs:
-                    nodes.add(str(out))
-                for dep in stage.deps:
-                    nodes.add(str(dep))
-            else:
-                nodes.add(stage.relpath)
-
-        edges = []
-
-        if outs:
-            for stage in networkx.dfs_preorder_nodes(G, target_stage):
-                for dep in stage.deps:
-                    for out in stage.outs:
-                        edges.append((str(out), str(dep)))
-        else:
-            for from_stage, to_stage in networkx.dfs_edges(G, target_stage):
-                if commands:
-                    if to_stage.cmd is None:
-                        continue
-                    edges.append((from_stage.cmd, to_stage.cmd))
-                else:
-                    edges.append((from_stage.relpath, to_stage.relpath))
-
-        return list(nodes), edges, networkx.is_tree(G)
-
-    def _show_ascii(self, target, commands, outs):
-        from dvc.dagascii import draw
-
-        nodes, edges, _ = self._build_graph(target, commands, outs)
-
-        if not nodes:
-            return
-
-        draw(nodes, edges)
-
-    def _show_dependencies_tree(self, target, commands, outs):
-        from treelib import Tree
-
-        nodes, edges, is_tree = self._build_graph(target, commands, outs)
-        if not nodes:
-            return
-        if not is_tree:
-            raise DvcException(
-                "DAG is not a tree, can not print it in tree-structure way, "
-                "please use --ascii instead"
-            )
-
-        tree = Tree()
-        tree.create_node(target, target)  # Root node
-        observe_list = [target]
-        while len(observe_list) > 0:
-            current_root = observe_list[0]
-            for edge in edges:
-                if edge[0] == current_root:
-                    tree.create_node(edge[1], edge[1], parent=current_root)
-                    observe_list.append(edge[1])
-            observe_list.pop(0)
-        tree.show()
-
-    def _write_dot(self, target, commands, outs):
-        import io
-        import networkx
-        from networkx.drawing.nx_pydot import write_dot
-
-        _, edges, _ = self._build_graph(target, commands, outs)
-        edges = [edge[::-1] for edge in edges]
-
-        simple_g = networkx.DiGraph()
-        simple_g.add_edges_from(edges)
-
-        dot_file = io.StringIO()
-        write_dot(simple_g, dot_file)
-        logger.info(dot_file.getvalue())
-
-    def run(self):
-        if not self.args.targets:
-            self.args.targets = self.default_targets
-
-        for target in self.args.targets:
-            try:
-                if self.args.ascii:
-                    self._show_ascii(
-                        target, self.args.commands, self.args.outs
-                    )
-                elif self.args.dot:
-                    self._write_dot(target, self.args.commands, self.args.outs)
-                elif self.args.tree:
-                    self._show_dependencies_tree(
-                        target, self.args.commands, self.args.outs
-                    )
-                else:
-                    self._show(
-                        target,
-                        self.args.commands,
-                        self.args.outs,
-                        self.args.locked,
-                    )
-            except DvcException:
-                msg = "failed to show pipeline for '{}'".format(target)
-                logger.exception(msg)
-                return 1
-        return 0
-
-
-class CmdPipelineList(CmdBase):
-    def run(self):
-        pipelines = self.repo.pipelines
-        for pipeline in pipelines:
-            for stage in pipeline:
-                logger.info(stage.relpath)
-            if len(pipeline) != 0:
-                logger.info("=" * 80)
-        logger.info("{} pipelines total".format(len(pipelines)))
-
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    PIPELINE_HELP = "Manage pipelines."
-    pipeline_parser = subparsers.add_parser(
-        "pipeline",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_HELP, "pipeline"),
-        help=PIPELINE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    pipeline_subparsers = pipeline_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc pipeline CMD --help` for command-specific help.",
-    )
-
-    fix_subparsers(pipeline_subparsers)
-
-    PIPELINE_SHOW_HELP = "Show pipelines."
-    pipeline_show_parser = pipeline_subparsers.add_parser(
-        "show",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_SHOW_HELP, "pipeline/show"),
-        help=PIPELINE_SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
-    pipeline_show_group.add_argument(
-        "-c",
-        "--commands",
-        action="store_true",
-        default=False,
-        help="Print commands instead of paths to DVC-files.",
-    )
-    pipeline_show_group.add_argument(
-        "-o",
-        "--outs",
-        action="store_true",
-        default=False,
-        help="Print output files instead of paths to DVC-files.",
-    )
-    pipeline_show_parser.add_argument(
-        "-l",
-        "--locked",
-        action="store_true",
-        default=False,
-        help="Print locked DVC stages",
-    )
-    pipeline_show_parser.add_argument(
-        "--ascii",
-        action="store_true",
-        default=False,
-        help="Output DAG as ASCII.",
-    )
-    pipeline_show_parser.add_argument(
-        "--dot",
-        action="store_true",
-        default=False,
-        help="Print DAG with .dot format.",
-    )
-    pipeline_show_parser.add_argument(
-        "--tree",
-        action="store_true",
-        default=False,
-        help="Output DAG as Dependencies Tree.",
-    )
-    pipeline_show_parser.add_argument(
-        "targets",
-        nargs="*",
-        help="DVC-files to show pipeline for. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-    )
-    pipeline_show_parser.set_defaults(func=CmdPipelineShow)
-
-    PIPELINE_LIST_HELP = "List pipelines."
-    pipeline_list_parser = pipeline_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(PIPELINE_LIST_HELP, "pipeline/list"),
-        help=PIPELINE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/plots.py

@@ -0,0 +1,245 @@
+import argparse
+import logging
+import os
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import DvcException
+from dvc.schema import PLOT_PROPS
+from dvc.utils import format_link
+
+logger = logging.getLogger(__name__)
+
+PAGE_HTML = """<!DOCTYPE html>
+<html>
+<head>
+    <title>DVC Plot</title>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+</head>
+<body>
+    {divs}
+</body>
+</html>"""
+
+DIV_HTML = """<div id = "{id}"></div>
+<script type = "text/javascript">
+    var spec = {vega_json};
+    vegaEmbed('#{id}', spec);
+</script>"""
+
+
+class CmdPlots(CmdBase):
+    def _func(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def _props(self):
+        # Pass only props specified by user, to not shadow ones from plot def
+        props = {p: getattr(self.args, p) for p in PLOT_PROPS}
+        return {k: v for k, v in props.items() if v is not None}
+
+    def run(self):
+        if self.args.show_vega:
+            if not self.args.targets:
+                logger.error("please specify a target for `--show-vega`")
+                return 1
+            if len(self.args.targets) > 1:
+                logger.error(
+                    "you can only specify one target for `--show-vega`"
+                )
+                return 1
+
+        try:
+            plots = self._func(targets=self.args.targets, props=self._props())
+
+            if self.args.show_vega:
+                target = self.args.targets[0]
+                logger.info(plots[target])
+                return 0
+
+            divs = [
+                DIV_HTML.format(id=f"plot{i}", vega_json=plot)
+                for i, plot in enumerate(plots.values())
+            ]
+            html = PAGE_HTML.format(divs="\n".join(divs))
+            path = self.args.out or "plots.html"
+
+            with open(path, "w") as fobj:
+                fobj.write(html)
+
+            logger.info(
+                "file://{}".format(os.path.join(self.repo.root_dir, path))
+            )
+
+        except DvcException:
+            logger.exception("")
+            return 1
+
+        return 0
+
+
+class CmdPlotsShow(CmdPlots):
+    def _func(self, *args, **kwargs):
+        return self.repo.plots.show(*args, **kwargs)
+
+
+class CmdPlotsDiff(CmdPlots):
+    def _func(self, *args, **kwargs):
+        return self.repo.plots.diff(
+            *args,
+            revs=self.args.revisions,
+            experiment=self.args.experiment,
+            **kwargs,
+        )
+
+
+class CmdPlotsModify(CmdPlots):
+    def run(self):
+        self.repo.plots.modify(
+            self.args.target, props=self._props(), unset=self.args.unset,
+        )
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    PLOTS_HELP = (
+        "Commands to visualize and compare plot metrics in structured files "
+        "(JSON, YAML, CSV, TSV)"
+    )
+
+    plots_parser = subparsers.add_parser(
+        "plots",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_HELP, "plots"),
+        help=PLOTS_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_subparsers = plots_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc plots CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(plots_subparsers)
+
+    SHOW_HELP = "Generate plots from metric files."
+    plots_show_parser = plots_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(SHOW_HELP, "plots/show"),
+        help=SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_show_parser.add_argument(
+        "targets",
+        nargs="*",
+        help="Plots files to visualize. Shows all plots by default.",
+    ).complete = completion.FILE
+    _add_props_arguments(plots_show_parser)
+    _add_output_arguments(plots_show_parser)
+    plots_show_parser.set_defaults(func=CmdPlotsShow)
+
+    PLOTS_DIFF_HELP = (
+        "Show multiple versions of plot metrics "
+        "by plotting them in a single image."
+    )
+    plots_diff_parser = plots_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_DIFF_HELP, "plots/diff"),
+        help=PLOTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help="Plots file to visualize. Shows all plots by default.",
+        metavar="<path>",
+    ).complete = completion.FILE
+    plots_diff_parser.add_argument(
+        "-e",
+        "--experiment",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    plots_diff_parser.add_argument(
+        "revisions", nargs="*", default=None, help="Git commits to plot from",
+    )
+    _add_props_arguments(plots_diff_parser)
+    _add_output_arguments(plots_diff_parser)
+    plots_diff_parser.set_defaults(func=CmdPlotsDiff)
+
+    PLOTS_MODIFY_HELP = "Modify display properties of plot metric files."
+    plots_modify_parser = plots_subparsers.add_parser(
+        "modify",
+        parents=[parent_parser],
+        description=append_doc_link(PLOTS_MODIFY_HELP, "plots/modify"),
+        help=PLOTS_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    plots_modify_parser.add_argument(
+        "target", help="Metric file to set properties to",
+    ).complete = completion.FILE
+    _add_props_arguments(plots_modify_parser)
+    plots_modify_parser.add_argument(
+        "--unset",
+        nargs="*",
+        metavar="<property>",
+        help="Unset one or more display properties.",
+    )
+    plots_modify_parser.set_defaults(func=CmdPlotsModify)
+
+
+def _add_props_arguments(parser):
+    parser.add_argument(
+        "-t",
+        "--template",
+        nargs="?",
+        default=None,
+        help=(
+            "Special JSON or HTML schema file to inject with the data. "
+            "See {}".format(
+                format_link("https://man.dvc.org/plots#plot-templates")
+            )
+        ),
+        metavar="<path>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-x", default=None, help="Field name for X axis.", metavar="<field>"
+    )
+    parser.add_argument(
+        "-y", default=None, help="Field name for Y axis.", metavar="<field>"
+    )
+    parser.add_argument(
+        "--no-header",
+        action="store_false",
+        dest="header",
+        default=None,  # Use default None to distinguish when it's not used
+        help="Provided CSV or TSV datafile does not have a header.",
+    )
+    parser.add_argument(
+        "--title", default=None, metavar="<text>", help="Plot title."
+    )
+    parser.add_argument(
+        "--x-label", default=None, help="X axis label", metavar="<text>"
+    )
+    parser.add_argument(
+        "--y-label", default=None, help="Y axis label", metavar="<text>"
+    )
+
+
+def _add_output_arguments(parser):
+    parser.add_argument(
+        "-o",
+        "--out",
+        default=None,
+        help="Destination path to save plots to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    parser.add_argument(
+        "--show-vega",
+        action="store_true",
+        default=False,
+        help="Show output in Vega format.",
+    )

dvc/command/remote.py

@@ -1,9 +1,9 @@
 import argparse
 import logging
 
-from dvc.config import ConfigError
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.config import ConfigError, merge
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -18,17 +18,13 @@ class CmdRemote(CmdConfig):
 
     def _check_exists(self, conf):
         if self.args.name not in conf["remote"]:
-            raise ConfigError(
-                "remote '{}' doesn't exists.".format(self.args.name)
-            )
+            raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
 
 
 class CmdRemoteAdd(CmdRemote):
     def run(self):
         if self.args.default:
-            logger.info(
-                "Setting '{}' as a default remote.".format(self.args.name)
-            )
+            logger.info(f"Setting '{self.args.name}' as a default remote.")
 
         with self.config.edit(self.args.level) as conf:
             if self.args.name in conf["remote"] and not self.args.force:
@@ -65,8 +61,17 @@ class CmdRemoteRemove(CmdRemote):
 class CmdRemoteModify(CmdRemote):
     def run(self):
         with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
-            conf["remote"][self.args.name][self.args.option] = self.args.value
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["remote"]:
+                conf["remote"][self.args.name] = {}
+            section = conf["remote"][self.args.name]
+            if self.args.unset:
+                section.pop(self.args.option, None)
+            else:
+                section[self.args.option] = self.args.value
         return 0
 
 
@@ -85,7 +90,18 @@ class CmdRemoteDefault(CmdRemote):
                 if self.args.unset:
                     conf["core"].pop("remote", None)
                 else:
-                    conf["core"]["remote"] = self.args.name
+                    merged_conf = self.config.load_config_to_level(
+                        self.args.level
+                    )
+                    if (
+                        self.args.name in conf["remote"]
+                        or self.args.name in merged_conf["remote"]
+                    ):
+                        conf["core"]["remote"] = self.args.name
+                    else:
+                        raise ConfigError(
+                            "default remote must be present in remote list."
+                        )
         return 0
 
 
@@ -97,10 +113,39 @@ class CmdRemoteList(CmdRemote):
         return 0
 
 
+class CmdRemoteRename(CmdRemote):
+    def _rename_default(self, conf):
+        if conf["core"].get("remote") == self.args.name:
+            conf["core"]["remote"] = self.args.new
+
+    def run(self):
+        all_config = self.config.load_config_to_level(None)
+        if self.args.new in all_config.get("remote", {}):
+            raise ConfigError(
+                "Rename failed. Remote name '{}' already exists.".format(
+                    {self.args.new}
+                )
+            )
+
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            conf["remote"][self.args.new] = conf["remote"][self.args.name]
+            del conf["remote"][self.args.name]
+            self._rename_default(conf)
+
+        for level in reversed(self.config.LEVELS):
+            if level == self.args.level:
+                break
+            with self.config.edit(level) as level_conf:
+                self._rename_default(level_conf)
+
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
-    REMOTE_HELP = "Manage remote storage configuration."
+    REMOTE_HELP = "Set up and manage data remotes."
     remote_parser = subparsers.add_parser(
         "remote",
         parents=[parent_parser],
@@ -116,7 +161,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(remote_subparsers)
 
-    REMOTE_ADD_HELP = "Add remote."
+    REMOTE_ADD_HELP = "Add a new data remote."
     remote_add_parser = remote_subparsers.add_parser(
         "add",
         parents=[parent_config_parser, parent_parser],
@@ -127,7 +172,7 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at {}".format(
+        help="Remote location. See full list of supported URLs at {}".format(
             format_link("https://man.dvc.org/remote")
         ),
     )
@@ -147,7 +192,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
-    REMOTE_DEFAULT_HELP = "Set/unset default remote."
+    REMOTE_DEFAULT_HELP = "Set/unset the default data remote."
     remote_default_parser = remote_subparsers.add_parser(
         "default",
         parents=[parent_config_parser, parent_parser],
@@ -156,7 +201,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_default_parser.add_argument(
-        "name", nargs="?", help="Name of the remote."
+        "name", nargs="?", help="Name of the remote"
     )
     remote_default_parser.add_argument(
         "-u",
@@ -167,20 +212,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_default_parser.set_defaults(func=CmdRemoteDefault)
 
-    REMOTE_REMOVE_HELP = "Remove remote."
-    remote_remove_parser = remote_subparsers.add_parser(
-        "remove",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
-        help=REMOTE_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    remote_remove_parser.add_argument(
-        "name", help="Name of the remote to remove."
-    )
-    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
-
-    REMOTE_MODIFY_HELP = "Modify remote."
+    REMOTE_MODIFY_HELP = "Modify the configuration of a data remote."
     remote_modify_parser = remote_subparsers.add_parser(
         "modify",
         parents=[parent_config_parser, parent_parser],
@@ -188,7 +220,7 @@ def add_parser(subparsers, parent_parser):
         help=REMOTE_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remote_modify_parser.add_argument("name", help="Name of the remote.")
+    remote_modify_parser.add_argument("name", help="Name of the remote")
     remote_modify_parser.add_argument(
         "option", help="Name of the option to modify."
     )
@@ -204,7 +236,7 @@ def add_parser(subparsers, parent_parser):
     )
     remote_modify_parser.set_defaults(func=CmdRemoteModify)
 
-    REMOTE_LIST_HELP = "List available remotes."
+    REMOTE_LIST_HELP = "List all available data remotes."
     remote_list_parser = remote_subparsers.add_parser(
         "list",
         parents=[parent_config_parser, parent_parser],
@@ -213,3 +245,27 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_list_parser.set_defaults(func=CmdRemoteList)
+
+    REMOTE_REMOVE_HELP = "Remove a data remote."
+    remote_remove_parser = remote_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_REMOVE_HELP, "remote/remove"),
+        help=REMOTE_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_remove_parser.add_argument(
+        "name", help="Name of the remote to remove."
+    )
+    remote_remove_parser.set_defaults(func=CmdRemoteRemove)
+    REMOTE_RENAME_HELP = "Rename a DVC remote"
+    remote_rename_parser = remote_subparsers.add_parser(
+        "rename",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(REMOTE_RENAME_HELP, "remote/rename"),
+        help=REMOTE_RENAME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    remote_rename_parser.add_argument("name", help="Remote to be renamed")
+    remote_rename_parser.add_argument("new", help="New name of the remote")
+    remote_rename_parser.set_defaults(func=CmdRemoteRename)

dvc/command/remove.py

@@ -1,48 +1,28 @@
 import argparse
 import logging
 
-import dvc.prompt as prompt
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdRemove(CmdBase):
-    def _is_outs_only(self, target):
-        if not self.args.purge:
-            return True
-
-        if self.args.force:
-            return False
-
-        msg = "Are you sure you want to remove '{}' with its outputs?".format(
-            target
-        )
-
-        if prompt.confirm(msg):
-            return False
-
-        raise DvcException(
-            "Cannot purge without a confirmation from the user."
-            " Use `-f` to force."
-        )
-
     def run(self):
         for target in self.args.targets:
             try:
-                outs_only = self._is_outs_only(target)
-                self.repo.remove(target, outs_only=outs_only)
+                self.repo.remove(target, outs=self.args.outs)
             except DvcException:
-                logger.exception("failed to remove '{}'".format(target))
+                logger.exception(f"failed to remove '{target}'")
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = "Remove DVC-file outputs."
+    REMOVE_HELP = (
+        "Remove stage entry, remove .gitignore entry and unprotect outputs"
+    )
     remove_parser = subparsers.add_parser(
         "remove",
         parents=[parent_parser],
@@ -50,29 +30,13 @@ def add_parser(subparsers, parent_parser):
         help=REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remove_parser_group = remove_parser.add_mutually_exclusive_group()
-    remove_parser_group.add_argument(
-        "-o",
+    remove_parser.add_argument(
         "--outs",
         action="store_true",
-        default=True,
-        help="Only remove DVC-file outputs. (Default)",
-    )
-    remove_parser_group.add_argument(
-        "-p",
-        "--purge",
-        action="store_true",
         default=False,
-        help="Remove DVC-file and all its outputs.",
+        help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Force purge.",
-    )
-    remove_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to remove."
-    )
+        "targets", nargs="+", help="DVC-files to remove.",
+    ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -2,13 +2,13 @@ import argparse
 import logging
 import os
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
+from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -35,10 +35,16 @@ class CmdRepro(CmdBase):
                     interactive=self.args.interactive,
                     pipeline=self.args.pipeline,
                     all_pipelines=self.args.all_pipelines,
-                    ignore_build_cache=self.args.ignore_build_cache,
+                    run_cache=not self.args.no_run_cache,
                     no_commit=self.args.no_commit,
                     downstream=self.args.downstream,
                     recursive=self.args.recursive,
+                    force_downstream=self.args.force_downstream,
+                    experiment=self.args.experiment,
+                    queue=self.args.queue,
+                    run_all=self.args.run_all,
+                    jobs=self.args.jobs,
+                    params=self.args.params,
                 )
 
                 if len(stages) == 0:
@@ -57,7 +63,9 @@ class CmdRepro(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    REPRO_HELP = "Check for changes and reproduce stages and dependencies."
+    REPRO_HELP = (
+        "Reproduce complete or partial pipelines by executing their stages."
+    )
     repro_parser = subparsers.add_parser(
         "repro",
         parents=[parent_parser],
@@ -68,8 +76,8 @@ def add_parser(subparsers, parent_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-file to reproduce. 'Dvcfile' by default.",
-    )
+        help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
+    ).complete = completion.DVC_FILE
     repro_parser.add_argument(
         "-f",
         "--force",
@@ -90,6 +98,7 @@ def add_parser(subparsers, parent_parser):
         "--cwd",
         default=os.path.curdir,
         help="Directory within your repo to reproduce from.",
+        metavar="<path>",
     )
     repro_parser.add_argument(
         "-m",
@@ -135,7 +144,16 @@ def add_parser(subparsers, parent_parser):
         help="Reproduce all stages in the specified directory.",
     )
     repro_parser.add_argument(
-        "--ignore-build-cache",
+        "--no-run-cache",
+        action="store_true",
+        default=False,
+        help=(
+            "Execute stage commands even if they have already been run with "
+            "the same command/dependencies/outputs/etc before."
+        ),
+    )
+    repro_parser.add_argument(
+        "--force-downstream",
         action="store_true",
         default=False,
         help="Reproduce all descendants of a changed stage even if their "
@@ -153,4 +171,30 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Start from the specified stages when reproducing pipelines.",
     )
+    repro_parser.add_argument(
+        "-e",
+        "--experiment",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    repro_parser.add_argument(
+        "--params",
+        action="append",
+        default=[],
+        help=argparse.SUPPRESS,
+        metavar="[<filename>:]<params_list>",
+    )
+    repro_parser.add_argument(
+        "--queue", action="store_true", default=False, help=argparse.SUPPRESS
+    )
+    repro_parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    repro_parser.add_argument(
+        "-j", "--jobs", type=int, help=argparse.SUPPRESS, metavar="<number>"
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -1,8 +1,7 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBaseNoRepo
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -17,7 +16,7 @@ class CmdRoot(CmdBaseNoRepo):
 
 
 def add_parser(subparsers, parent_parser):
-    ROOT_HELP = "Relative path to the repository's directory."
+    ROOT_HELP = "Return the relative path to the root of the DVC project."
     root_parser = subparsers.add_parser(
         "root",
         parents=[parent_parser],

dvc/command/run.py

@@ -1,18 +1,14 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdRun(CmdBase):
     def run(self):
-        overwrite = self.args.yes or self.args.overwrite_dvcfile
-
         if not any(
             [
                 self.args.deps,
@@ -20,15 +16,18 @@ class CmdRun(CmdBase):
                 self.args.outs_no_cache,
                 self.args.metrics,
                 self.args.metrics_no_cache,
+                self.args.plots,
+                self.args.plots_no_cache,
                 self.args.outs_persist,
                 self.args.outs_persist_no_cache,
+                self.args.params,
                 self.args.command,
             ]
         ):  # pragma: no cover
             logger.error(
                 "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
-                "`-m`, `-M`, `--outs-persist`, `--outs-persist-no-cache`, "
-                "`command`."
+                "`-m`, `-M`, `-p`, `--plots`, `--plots-no-cache`, "
+                "`--outs-persist`, `--outs-persist-no-cache`, `command`."
             )
             return 1
 
@@ -39,21 +38,25 @@ class CmdRun(CmdBase):
                 outs_no_cache=self.args.outs_no_cache,
                 metrics=self.args.metrics,
                 metrics_no_cache=self.args.metrics_no_cache,
+                plots=self.args.plots,
+                plots_no_cache=self.args.plots_no_cache,
                 deps=self.args.deps,
+                params=self.args.params,
                 fname=self.args.file,
-                cwd=self.args.cwd,
                 wdir=self.args.wdir,
                 no_exec=self.args.no_exec,
-                overwrite=overwrite,
-                ignore_build_cache=self.args.ignore_build_cache,
-                remove_outs=self.args.remove_outs,
+                force=self.args.force,
+                run_cache=not self.args.no_run_cache,
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
                 outs_persist_no_cache=self.args.outs_persist_no_cache,
                 always_changed=self.args.always_changed,
+                name=self.args.name,
+                single_stage=self.args.single_stage,
+                external=self.args.external,
             )
         except DvcException:
-            logger.exception("failed to run command")
+            logger.exception("")
             return 1
 
         return 0
@@ -77,7 +80,7 @@ class CmdRun(CmdBase):
         if " " not in argument or '"' in argument:
             return argument
 
-        return '"{}"'.format(argument)
+        return f'"{argument}"'
 
 
 def add_parser(subparsers, parent_parser):
@@ -95,6 +98,10 @@ def add_parser(subparsers, parent_parser):
         action="append",
         default=[],
         help="Declare dependencies for reproducible cmd.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-n", "--name", help="Stage name.",
     )
     run_parser.add_argument(
         "-o",
@@ -102,6 +109,7 @@ def add_parser(subparsers, parent_parser):
         action="append",
         default=[],
         help="Declare output file or directory.",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "-O",
@@ -110,32 +118,54 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory "
         "(do not put into DVC cache).",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "-p",
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter to use as additional dependency.",
+        metavar="[<filename>:]<params_list>",
     )
     run_parser.add_argument(
         "-m",
         "--metrics",
         action="append",
         default=[],
-        help="Declare output metric file or directory.",
+        help="Declare output metric file.",
+        metavar="<path>",
     )
     run_parser.add_argument(
         "-M",
         "--metrics-no-cache",
         action="append",
         default=[],
-        help="Declare output metric file or directory "
-        "(do not put into DVC cache).",
+        help="Declare output metric file (do not put into DVC cache).",
+        metavar="<path>",
     )
     run_parser.add_argument(
-        "-f", "--file", help="Specify name of the DVC-file it generates."
+        "--plots",
+        action="append",
+        default=[],
+        help="Declare output plot file.",
+        metavar="<path>",
     )
     run_parser.add_argument(
-        "-c", "--cwd", help="Deprecated, use -w and -f instead."
+        "--plots-no-cache",
+        action="append",
+        default=[],
+        help="Declare output plot file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "--file", metavar="<filename>", help=argparse.SUPPRESS,
     )
     run_parser.add_argument(
         "-w",
         "--wdir",
         help="Directory within your repo to run your command in.",
+        metavar="<path>",
     )
     run_parser.add_argument(
         "--no-exec",
@@ -144,30 +174,20 @@ def add_parser(subparsers, parent_parser):
         help="Only create stage file without actually running it.",
     )
     run_parser.add_argument(
-        "-y",
-        "--yes",
+        "-f",
+        "--force",
         action="store_true",
         default=False,
-        help="Deprecated, use --overwrite-dvcfile instead",
+        help="Overwrite existing stage",
     )
     run_parser.add_argument(
-        "--overwrite-dvcfile",
+        "--no-run-cache",
         action="store_true",
         default=False,
-        help="Overwrite existing DVC-file without asking for confirmation.",
-    )
-    run_parser.add_argument(
-        "--ignore-build-cache",
-        action="store_true",
-        default=False,
-        help="Run this stage even if it has been already ran with the same "
-        "command/dependencies/outputs/etc before.",
-    )
-    run_parser.add_argument(
-        "--remove-outs",
-        action="store_true",
-        default=False,
-        help="Deprecated, this is now the default behavior",
+        help=(
+            "Execute the command even if this stage has already been run "
+            "with the same command/dependencies/outputs/etc before."
+        ),
     )
     run_parser.add_argument(
         "--no-commit",
@@ -181,6 +201,7 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory that will not be "
         "removed upon repro.",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "--outs-persist-no-cache",
@@ -188,6 +209,7 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory that will not be "
         "removed upon repro (do not put into DVC cache).",
+        metavar="<filename>",
     )
     run_parser.add_argument(
         "--always-changed",
@@ -195,6 +217,18 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Always consider this DVC-file as changed.",
     )
+    run_parser.add_argument(
+        "--single-stage",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    run_parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow outputs that are outside of the DVC repository.",
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -1,7 +1,7 @@
 import logging
 
 from dvc.command.data_sync import CmdDataBase
-
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
@@ -20,7 +20,7 @@ class CmdDataStatus(CmdDataBase):
         ind = indent * self.STATUS_INDENT
 
         if isinstance(status, str):
-            logger.info("{}{}".format(ind, status))
+            logger.info(f"{ind}{status}")
             return
 
         if isinstance(status, list):
@@ -34,7 +34,7 @@ class CmdDataStatus(CmdDataBase):
             if isinstance(value, str):
                 logger.info("{}{}{}".format(ind, self._normalize(value), key))
             elif value:
-                logger.info("{}{}:".format(ind, key))
+                logger.info(f"{ind}{key}:")
                 self._show(value, indent + 1)
 
     def run(self):
@@ -47,17 +47,24 @@ class CmdDataStatus(CmdDataBase):
                 remote=self.args.remote,
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
                 with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
             )
-            if st:
-                if self.args.quiet:
-                    return 1
-                else:
-                    self._show(st, indent)
+
+            if self.args.quiet:
+                return bool(st)
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(st))
+            elif st:
+                self._show(st, indent)
             else:
                 logger.info(self.UP_TO_DATE_MSG)
 
-        except Exception:
+        except DvcException:
             logger.exception("failed to obtain data status")
             return 1
         return 0

dvc/command/tag.py

@@ -1,156 +0,0 @@
-import logging
-
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
-from dvc.command.base import fix_subparsers
-from dvc.exceptions import DvcException
-from dvc.utils import to_yaml_string
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdTagAdd(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                self.repo.tag.add(
-                    self.args.tag,
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-            except DvcException:
-                logger.exception("failed to add tag")
-                return 1
-        return 0
-
-
-class CmdTagRemove(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                self.repo.tag.remove(
-                    self.args.tag,
-                    target=target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-            except DvcException:
-                logger.exception("failed to remove tag")
-                return 1
-        return 0
-
-
-class CmdTagList(CmdBase):
-    def run(self):
-        for target in self.args.targets:
-            try:
-                tags = self.repo.tag.list(
-                    target,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-                if tags:
-                    logger.info(to_yaml_string(tags))
-            except DvcException:
-                logger.exception("failed list tags")
-                return 1
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    TAG_HELP = "A set of commands to manage DVC tags."
-    tag_parser = subparsers.add_parser(
-        "tag",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_HELP, "tag"),
-        add_help=False,
-    )
-
-    tag_subparsers = tag_parser.add_subparsers(
-        dest="cmd",
-        help="Use DVC tag CMD --help to display command-specific help.",
-    )
-
-    fix_subparsers(tag_subparsers)
-
-    TAG_ADD_HELP = "Add DVC tag."
-    tag_add_parser = tag_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_ADD_HELP, "tag-add"),
-        help=TAG_ADD_HELP,
-    )
-    tag_add_parser.add_argument("tag", help="Dvc tag.")
-    tag_add_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_add_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Add tag for all dependencies of the specified DVC-file.",
-    )
-    tag_add_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Add tag for subdirectories of the specified directory.",
-    )
-    tag_add_parser.set_defaults(func=CmdTagAdd)
-
-    TAG_REMOVE_HELP = "Remove DVC tag."
-    tag_remove_parser = tag_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
-        help=TAG_REMOVE_HELP,
-    )
-    tag_remove_parser.add_argument("tag", help="Dvc tag.")
-    tag_remove_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_remove_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="Remove tag for all dependencies of the specified DVC-file.",
-    )
-    tag_remove_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="Remove tag for subdirectories of the specified directory.",
-    )
-    tag_remove_parser.set_defaults(func=CmdTagRemove)
-
-    TAG_LIST_HELP = "List DVC tags."
-    tag_list_parser = tag_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(TAG_LIST_HELP, "tag-list"),
-        help=TAG_LIST_HELP,
-    )
-    tag_list_parser.add_argument(
-        "targets", nargs="*", default=[None], help="DVC-files."
-    )
-    tag_list_parser.add_argument(
-        "-d",
-        "--with-deps",
-        action="store_true",
-        default=False,
-        help="List tags for all dependencies of the specified DVC-file.",
-    )
-    tag_list_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="List tags for subdirectories of the specified directory.",
-    )
-    tag_list_parser.set_defaults(func=CmdTagList)

dvc/command/unprotect.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -15,14 +14,17 @@ class CmdUnprotect(CmdBase):
             try:
                 self.repo.unprotect(target)
             except DvcException:
-                msg = "failed to unprotect '{}'".format(target)
+                msg = f"failed to unprotect '{target}'"
                 logger.exception(msg)
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = "Unprotect data files or directories."
+    UNPROTECT_HELP = (
+        "Unprotect tracked files or directories (when hardlinks or symlinks "
+        "have been enabled with `dvc config cache.type`)"
+    )
     unprotect_parser = subparsers.add_parser(
         "unprotect",
         parents=[parent_parser],
@@ -31,6 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect."
-    )
+        "targets", nargs="+", help="Data files/directories to unprotect.",
+    ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -1,23 +1,25 @@
 import argparse
 import logging
 
-from dvc.command.base import append_doc_link
-from dvc.command.base import CmdBase
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
-
 logger = logging.getLogger(__name__)
 
 
 class CmdUpdate(CmdBase):
     def run(self):
         ret = 0
-        for target in self.args.targets:
-            try:
-                self.repo.update(target, self.args.rev)
-            except DvcException:
-                logger.exception("failed to update '{}'.".format(target))
-                ret = 1
+        try:
+            self.repo.update(
+                targets=self.args.targets,
+                rev=self.args.rev,
+                recursive=self.args.recursive,
+            )
+        except DvcException:
+            logger.exception("failed update data")
+            ret = 1
         return ret
 
 
@@ -31,9 +33,19 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to update."
+        "targets", nargs="+", help="DVC-files to update.",
+    ).complete = completion.DVC_FILE
+    update_parser.add_argument(
+        "--rev",
+        nargs="?",
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
     )
     update_parser.add_argument(
-        "--rev", nargs="?", help="Git revision (e.g. SHA, branch, tag)"
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Update all stages in the specified directory.",
     )
     update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -1,121 +1,23 @@
 import argparse
-import itertools
 import logging
-import os
-import platform
-import uuid
-import pathlib
 
-try:
-    import psutil
-except ImportError:
-    psutil = None
-
-from dvc.utils import is_binary, relpath
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.version import __version__
-from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.system import System
-from dvc.utils.pkg import PKG
+from dvc.info import get_dvc_info
 
 logger = logging.getLogger(__name__)
 
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
-        from dvc.repo import Repo
-
-        info = [
-            "DVC version: {}".format(__version__),
-            "Python version: {}".format(platform.python_version()),
-            "Platform: {}".format(platform.platform()),
-            "Binary: {}".format(is_binary()),
-            "Package: {}".format(PKG),
-        ]
-
-        try:
-            repo = Repo()
-            root_directory = repo.root_dir
-
-            # cache_dir might not exist yet (e.g. after `dvc init`), and we
-            # can't auto-create it, as it might cause issues if the user
-            # later decides to enable shared cache mode with
-            # `dvc config cache.shared group`.
-            if os.path.exists(repo.cache.local.cache_dir):
-                info.append(
-                    "Cache: {}".format(self.get_linktype_support_info(repo))
-                )
-                if psutil:
-                    fs_type = self.get_fs_type(repo.cache.local.cache_dir)
-                    info.append(
-                        "Filesystem type (cache directory): {}".format(fs_type)
-                    )
-            else:
-                logger.warning(
-                    "Unable to detect supported link types, as cache "
-                    "directory '{}' doesn't exist. It is usually auto-created "
-                    "by commands such as `dvc add/fetch/pull/run/import`, "
-                    "but you could create it manually to enable this "
-                    "check.".format(relpath(repo.cache.local.cache_dir))
-                )
-
-        except NotDvcRepoError:
-            root_directory = os.getcwd()
-
-        if psutil:
-            fs_root = self.get_fs_type(os.path.abspath(root_directory))
-            info.append("Filesystem type (workspace): {}".format(fs_root))
-
-        logger.info("\n".join(info))
+        dvc_info = get_dvc_info()
+        logger.info(dvc_info)
         return 0
 
-    @staticmethod
-    def get_fs_type(path):
-        partition = {
-            pathlib.Path(part.mountpoint): (part.fstype, part.device)
-            for part in psutil.disk_partitions(all=True)
-        }
-
-        path = pathlib.Path(path)
-
-        for parent in itertools.chain([path], path.parents):
-            if parent in partition:
-                return partition[parent]
-        return ("unknown", "none")
-
-    @staticmethod
-    def get_linktype_support_info(repo):
-        links = {
-            "reflink": (System.reflink, None),
-            "hardlink": (System.hardlink, System.is_hardlink),
-            "symlink": (System.symlink, System.is_symlink),
-        }
-
-        fname = "." + str(uuid.uuid4())
-        src = os.path.join(repo.cache.local.cache_dir, fname)
-        open(src, "w").close()
-        dst = os.path.join(repo.root_dir, fname)
-
-        cache = []
-
-        for name, (link, is_link) in links.items():
-            try:
-                link(src, dst)
-                status = "supported"
-                if is_link and not is_link(dst):
-                    status = "broken"
-                os.unlink(dst)
-            except DvcException:
-                status = "not supported"
-            cache.append("{name} - {status}".format(name=name, status=status))
-        os.remove(src)
-
-        return ", ".join(cache)
-
 
 def add_parser(subparsers, parent_parser):
-    VERSION_HELP = "Show DVC version and system/environment information."
-
+    VERSION_HELP = (
+        "Display the DVC version and system/environment information."
+    )
     version_parser = subparsers.add_parser(
         "version",
         parents=[parent_parser],

dvc/compat.py

@@ -1,42 +0,0 @@
-"""Python version compatibility code"""
-
-# Backport os.fspath() from Python 3.6
-try:
-    from os import fspath  # noqa: F821
-
-    fspath_py35 = lambda s: s  # noqa: E731
-except ImportError:
-
-    def fspath(path):
-        """Return the path representation of a path-like object.
-
-        If str or bytes is passed in, it is returned unchanged. Otherwise the
-        os.PathLike interface is used to get the path representation. If the
-        path representation is not str or bytes, TypeError is raised. If the
-        provided path is not str, bytes, or os.PathLike, TypeError is raised.
-        """
-        if isinstance(path, (str, bytes)):
-            return path
-
-        # Work from the object's type to match method resolution of other magic
-        # methods.
-        path_type = type(path)
-        try:
-            path_repr = path_type.__fspath__(path)
-        except AttributeError:
-            if hasattr(path_type, "__fspath__"):
-                raise
-            else:
-                raise TypeError(
-                    "expected str, bytes or os.PathLike object, "
-                    "not " + path_type.__name__
-                )
-        if isinstance(path_repr, (str, bytes)):
-            return path_repr
-        else:
-            raise TypeError(
-                "expected {}.__fspath__() to return str or bytes, "
-                "not {}".format(path_type.__name__, type(path_repr).__name__)
-            )
-
-    fspath_py35 = fspath

dvc/config.py

@@ -1,16 +1,27 @@
 """DVC config objects."""
-from contextlib import contextmanager
 import logging
 import os
 import re
+from contextlib import contextmanager
+from functools import partial
 from urllib.parse import urlparse
 
-from funcy import cached_property, re_find, walk_values, compact
 import configobj
-from voluptuous import Schema, Optional, Invalid, ALLOW_EXTRA
-from voluptuous import All, Any, Lower, Range, Coerce
+from funcy import cached_property, compact, re_find, walk_values
+from voluptuous import (
+    ALLOW_EXTRA,
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
 
 from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.path_info import PathInfo
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -20,7 +31,7 @@ class ConfigError(DvcException):
     """DVC config exception."""
 
     def __init__(self, msg):
-        super().__init__("config file error: {}".format(msg))
+        super().__init__(f"config file error: {msg}")
 
 
 class NoRemoteError(ConfigError):
@@ -78,7 +89,7 @@ def ByUrl(mapping):
         if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
             return schemas[""](data)
         if parsed.scheme not in schemas:
-            raise Invalid("Unsupported URL type {}://".format(parsed.scheme))
+            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
 
         return schemas[parsed.scheme](data)
 
@@ -92,15 +103,34 @@ class RelPath(str):
 REMOTE_COMMON = {
     "url": str,
     "checksum_jobs": All(Coerce(int), Range(1)),
-    "no_traverse": Bool,
+    Optional("no_traverse"): Bool,  # obsoleted
     "verify": Bool,
 }
 LOCAL_COMMON = {
     "type": supported_cache_type,
-    Optional("protected", default=False): Bool,
+    Optional("protected", default=False): Bool,  # obsoleted
     "shared": All(Lower, Choices("group")),
     Optional("slow_link_warning", default=True): Bool,
 }
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "ssl_verify": Bool,
+    "method": str,
+}
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+}
+
 SCHEMA = {
     "core": {
         "remote": Lower,
@@ -108,6 +138,9 @@ SCHEMA = {
         Optional("interactive", default=False): Bool,
         Optional("analytics", default=True): Bool,
         Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+        Optional("experiments", default=False): Bool,
+        Optional("check_update", default=True): Bool,
     },
     "cache": {
         "local": str,
@@ -129,9 +162,12 @@ SCHEMA = {
                     "profile": str,
                     "credentialpath": str,
                     "endpointurl": str,
-                    Optional("listobjects", default=False): Bool,
+                    "access_key_id": str,
+                    "secret_access_key": str,
+                    Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
                     "sse": str,
+                    "sse_kms_key_id": str,
                     "acl": str,
                     "grant_read": str,
                     "grant_read_acp": str,
@@ -164,13 +200,20 @@ SCHEMA = {
                     **REMOTE_COMMON,
                 },
                 "gdrive": {
+                    "gdrive_use_service_account": Bool,
                     "gdrive_client_id": str,
                     "gdrive_client_secret": str,
                     "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_email": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_p12_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
                     **REMOTE_COMMON,
                 },
-                "http": REMOTE_COMMON,
-                "https": REMOTE_COMMON,
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
                 "remote": {str: object},  # Any of the above options are valid
             }
         )
@@ -193,9 +236,8 @@ class Config(dict):
         validate (bool): optional flag to tell dvc if it should validate the
             config or just load it as is. 'True' by default.
 
-
     Raises:
-        ConfigError: thrown when config has an invalid format.
+        ConfigError: thrown if config has an invalid format.
     """
 
     APPNAME = "dvc"
@@ -207,7 +249,11 @@ class Config(dict):
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
-    def __init__(self, dvc_dir=None, validate=True):
+    def __init__(
+        self, dvc_dir=None, validate=True, tree=None,
+    ):  # pylint: disable=super-init-not-called
+        from dvc.tree.local import LocalTree
+
         self.dvc_dir = dvc_dir
 
         if not dvc_dir:
@@ -220,11 +266,14 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
+        self.wtree = LocalTree(None, {"url": self.dvc_dir})
+        self.tree = tree or self.wtree
+
         self.load(validate=validate)
 
     @classmethod
     def get_dir(cls, level):
-        from appdirs import user_config_dir, site_config_dir
+        from appdirs import site_config_dir, user_config_dir
 
         assert level in ("global", "system")
 
@@ -264,12 +313,9 @@ class Config(dict):
         """Loads config from all the config files.
 
         Raises:
-            dvc.config.ConfigError: thrown if config has invalid format.
+            ConfigError: thrown if config has an invalid format.
         """
-        conf = {}
-        for level in self.LEVELS:
-            if level in self.files:
-                _merge(conf, self.load_one(level))
+        conf = self.load_config_to_level()
 
         if validate:
             conf = self.validate(conf)
@@ -281,11 +327,40 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
+    def _get_tree(self, level):
+        # NOTE: this might be a GitTree, which doesn't see things outside of
+        # the repo.
+        return self.tree if level == "repo" else self.wtree
+
+    def _load_config(self, level):
+        filename = self.files[level]
+        tree = self._get_tree(level)
+
+        if tree.exists(filename, use_dvcignore=False):
+            with tree.open(filename) as fobj:
+                conf_obj = configobj.ConfigObj(fobj)
+        else:
+            conf_obj = configobj.ConfigObj()
+        return _parse_remotes(_lower_keys(conf_obj.dict()))
+
+    def _save_config(self, level, conf_dict):
+        filename = self.files[level]
+        tree = self._get_tree(level)
+
+        logger.debug(f"Writing '{filename}'.")
+
+        tree.makedirs(os.path.dirname(filename))
+
+        config = configobj.ConfigObj(_pack_remotes(conf_dict))
+        with tree.open(filename, "wb") as fobj:
+            config.write(fobj)
+        config.filename = filename
+
     def load_one(self, level):
-        conf = _load_config(self.files[level])
+        conf = self._load_config(level)
         conf = self._load_paths(conf, self.files[level])
 
-        # Autovivify sections
+        # Auto-verify sections
         for key in COMPILED_SCHEMA.schema:
             conf.setdefault(key, {})
 
@@ -302,35 +377,57 @@ class Config(dict):
 
         return Config._map_dirs(conf, resolve)
 
+    @staticmethod
+    def _to_relpath(conf_dir, path):
+        if re.match(r"\w+://", path):
+            return path
+
+        if isinstance(path, RelPath) or not os.path.isabs(path):
+            path = relpath(path, conf_dir)
+
+        return PathInfo(path).as_posix()
+
     @staticmethod
     def _save_paths(conf, filename):
         conf_dir = os.path.dirname(filename)
-
-        def rel(path):
-            if re.match(r"\w+://", path):
-                return path
-
-            if isinstance(path, RelPath) or not os.path.isabs(path):
-                return relpath(path, conf_dir)
-            return path
+        rel = partial(Config._to_relpath, conf_dir)
 
         return Config._map_dirs(conf, rel)
 
     @staticmethod
     def _map_dirs(conf, func):
-        dirs_schema = {"cache": {"dir": func}, "remote": {str: {"url": func}}}
+        dirs_schema = {
+            "cache": {"dir": func},
+            "remote": {
+                str: {"url": func, "gdrive_user_credentials_file": func}
+            },
+        }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
 
+    def load_config_to_level(self, level=None):
+        merged_conf = {}
+        for merge_level in self.LEVELS:
+            if merge_level == level:
+                break
+            if merge_level in self.files:
+                merge(merged_conf, self.load_one(merge_level))
+        return merged_conf
+
     @contextmanager
     def edit(self, level="repo"):
         if level in {"repo", "local"} and self.dvc_dir is None:
-            raise ConfigError("Not inside a dvc repo")
+            raise ConfigError("Not inside a DVC repo")
 
         conf = self.load_one(level)
         yield conf
 
         conf = self._save_paths(conf, self.files[level])
-        _save_config(self.files[level], conf)
+
+        merged_conf = self.load_config_to_level(level)
+        merge(merged_conf, conf)
+        self.validate(merged_conf)
+
+        self._save_config(level, conf)
         self.load()
 
     @staticmethod
@@ -341,20 +438,6 @@ class Config(dict):
             raise ConfigError(str(exc)) from None
 
 
-def _load_config(filename):
-    conf_obj = configobj.ConfigObj(filename)
-    return _parse_remotes(_lower_keys(conf_obj.dict()))
-
-
-def _save_config(filename, conf_dict):
-    logger.debug("Writing '{}'.".format(filename))
-    os.makedirs(os.path.dirname(filename), exist_ok=True)
-
-    config = configobj.ConfigObj(_pack_remotes(conf_dict))
-    config.filename = filename
-    config.write()
-
-
 def _parse_remotes(conf):
     result = {"remote": {}}
 
@@ -374,17 +457,17 @@ def _pack_remotes(conf):
 
     # Transform remote.name -> 'remote "name"'
     for name, val in conf["remote"].items():
-        result['remote "{}"'.format(name)] = val
+        result[f'remote "{name}"'] = val
     result.pop("remote", None)
 
     return result
 
 
-def _merge(into, update):
+def merge(into, update):
     """Merges second dict into first recursively"""
     for key, val in update.items():
         if isinstance(into.get(key), dict) and isinstance(val, dict):
-            _merge(into[key], val)
+            merge(into[key], val)
         else:
             into[key] = val
 

dvc/daemon.py

@@ -3,13 +3,12 @@
 import inspect
 import logging
 import os
+import platform
 import sys
 from subprocess import Popen
 
 from dvc.env import DVC_DAEMON
-from dvc.utils import fix_env
-from dvc.utils import is_binary
-
+from dvc.utils import fix_env, is_binary
 
 logger = logging.getLogger(__name__)
 
@@ -17,25 +16,30 @@ CREATE_NEW_PROCESS_GROUP = 0x00000200
 DETACHED_PROCESS = 0x00000008
 
 
+def _popen(cmd, **kwargs):
+    prefix = [sys.executable]
+    if not is_binary():
+        prefix += [sys.argv[0]]
+
+    return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
+
+
 def _spawn_windows(cmd, env):
-    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
 
     creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    Popen(
-        cmd,
-        env=env,
-        close_fds=True,
-        shell=False,
-        creationflags=creationflags,
-        startupinfo=startupinfo,
+    _popen(
+        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
     ).communicate()
 
 
 def _spawn_posix(cmd, env):
+    from dvc.main import main
+
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
@@ -48,7 +52,6 @@ def _spawn_posix(cmd, env):
         os._exit(1)  # pylint: disable=protected-access
 
     os.setsid()
-    os.umask(0)
 
     try:
         pid = os.fork()
@@ -62,13 +65,19 @@ def _spawn_posix(cmd, env):
     sys.stdout.close()
     sys.stderr.close()
 
-    Popen(cmd, env=env, close_fds=True, shell=False).communicate()
+    if platform.system() == "Darwin":
+        # workaround for MacOS bug
+        # https://github.com/iterative/dvc/issues/4294
+        _popen(cmd, env=env).communicate()
+    else:
+        os.environ.update(env)
+        main(cmd)
 
     os._exit(0)  # pylint: disable=protected-access
 
 
 def _spawn(cmd, env):
-    logger.debug("Trying to spawn '{}'".format(cmd))
+    logger.debug(f"Trying to spawn '{cmd}'")
 
     if os.name == "nt":
         _spawn_windows(cmd, env)
@@ -77,7 +86,7 @@ def _spawn(cmd, env):
     else:
         raise NotImplementedError
 
-    logger.debug("Spawned '{}'".format(cmd))
+    logger.debug(f"Spawned '{cmd}'")
 
 
 def daemon(args):
@@ -90,10 +99,7 @@ def daemon(args):
         logger.debug("skipping launching a new daemon.")
         return
 
-    cmd = [sys.executable]
-    if not is_binary():
-        cmd += [sys.argv[0]]
-    cmd += ["daemon", "-q"] + args
+    cmd = ["daemon", "-q"] + args
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])

dvc/dagascii.py

@@ -6,12 +6,9 @@ import os
 import pydoc
 import sys
 
-from grandalf.graphs import Edge
-from grandalf.graphs import Graph
-from grandalf.graphs import Vertex
+from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
-from grandalf.routing import EdgeViewer
-from grandalf.routing import route_with_lines
+from grandalf.routing import EdgeViewer, route_with_lines
 
 from dvc.env import DVC_PAGER
 from dvc.utils import format_link
@@ -40,7 +37,7 @@ def find_pager():
     if env_pager:
         return make_pager(env_pager)
 
-    if os.system("({}) 2>{}".format(DEFAULT_PAGER, os.devnull)) == 0:
+    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
         return make_pager(DEFAULT_PAGER_FORMATTED)
 
     logger.warning(
@@ -52,7 +49,7 @@ def find_pager():
     return pydoc.plainpager
 
 
-class VertexViewer(object):
+class VertexViewer:
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.
 
@@ -78,7 +75,7 @@ class VertexViewer(object):
         return self._w
 
 
-class AsciiCanvas(object):
+class AsciiCanvas:
     """Class for drawing in ASCII.
 
     Args:
@@ -95,14 +92,13 @@ class AsciiCanvas(object):
         self.cols = cols
         self.lines = lines
 
-        self.canvas = [[" "] * cols for l in range(lines)]
+        self.canvas = [[" "] * cols for line in range(lines)]
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
-        pager = find_pager()
         lines = map("".join, self.canvas)
         joined_lines = os.linesep.join(lines)
-        pager(joined_lines)
+        return joined_lines
 
     def point(self, x, y, char):
         """Create a point on ASCII canvas.
@@ -148,21 +144,21 @@ class AsciiCanvas(object):
                 if dx == 0:
                     y = y0
                 else:
-                    y = y0 + int(round((x - x0) * dy / float((dx))))
+                    y = y0 + int(round((x - x0) * dy / float(dx)))
                 self.point(x, y, char)
         elif y0 < y1:
             for y in range(y0, y1 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x0 + int(round((y - y0) * dx / float((dy))))
+                    x = x0 + int(round((y - y0) * dx / float(dy)))
                 self.point(x, y, char)
         else:
             for y in range(y1, y0 + 1):
                 if dy == 0:
                     x = x0
                 else:
-                    x = x1 + int(round((y - y1) * dx / float((dy))))
+                    x = x1 + int(round((y - y1) * dx / float(dy)))
                 self.point(x, y, char)
 
     def text(self, x, y, text):
@@ -216,7 +212,7 @@ def _build_sugiyama_layout(vertexes, edges):
     # Y
     #
 
-    vertexes = {v: Vertex(" {} ".format(v)) for v in vertexes}
+    vertexes = {v: Vertex(f" {v} ") for v in vertexes}
     # NOTE: reverting edges to correctly orientate the graph
     edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
     vertexes = vertexes.values()
@@ -226,7 +222,7 @@ def _build_sugiyama_layout(vertexes, edges):
         vertex.view = VertexViewer(vertex.data)
 
     # NOTE: determine min box length to create the best layout
-    minw = min([v.view.w for v in vertexes])
+    minw = min(v.view.w for v in vertexes)
 
     for edge in edges:
         edge.view = EdgeViewer()
@@ -319,4 +315,4 @@ def draw(vertexes, edges):
             int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
         )
 
-    canvas.draw()
+    return canvas.draw()

dvc/data_cloud.py

@@ -3,13 +3,12 @@
 import logging
 
 from dvc.config import NoRemoteError
-from dvc.remote import Remote
-
+from dvc.remote import get_remote
 
 logger = logging.getLogger(__name__)
 
 
-class DataCloud(object):
+class DataCloud:
     """Class that manages dvc remotes.
 
     Args:
@@ -23,12 +22,12 @@ class DataCloud(object):
     def __init__(self, repo):
         self.repo = repo
 
-    def get_remote(self, remote=None, command="<command>"):
-        if not remote:
-            remote = self.repo.config["core"].get("remote")
+    def get_remote(self, name=None, command="<command>"):
+        if not name:
+            name = self.repo.config["core"].get("remote")
 
-        if remote:
-            return self._init_remote(remote)
+        if name:
+            return self._init_remote(name)
 
         if bool(self.repo.config["remote"]):
             error_msg = (
@@ -45,70 +44,77 @@ class DataCloud(object):
 
         raise NoRemoteError(error_msg)
 
-    def _init_remote(self, remote):
-        return Remote(self.repo, name=remote)
+    def _init_remote(self, name):
+        return get_remote(self.repo, name=name)
 
-    def push(self, cache, jobs=None, remote=None, show_checksums=False):
+    def push(
+        self, cache, jobs=None, remote=None, show_checksums=False,
+    ):
         """Push data items in a cloud-agnostic way.
 
         Args:
             cache (NamedCache): named checksums to push to the cloud.
             jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to push to.
+            remote (dvc.remote.base.BaseRemote): optional remote to push to.
                 By default remote from core.remote config option is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
+        remote = self.get_remote(remote, "push")
+
         return self.repo.cache.local.push(
-            cache,
-            jobs=jobs,
-            remote=self.get_remote(remote, "push"),
-            show_checksums=show_checksums,
+            cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
         )
 
-    def pull(self, cache, jobs=None, remote=None, show_checksums=False):
+    def pull(
+        self, cache, jobs=None, remote=None, show_checksums=False,
+    ):
         """Pull data items in a cloud-agnostic way.
 
         Args:
             cache (NamedCache): named checksums to pull from the cloud.
             jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
+            remote (dvc.remote.base.BaseRemote): optional remote to pull from.
                 By default remote from core.remote config option is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
         remote = self.get_remote(remote, "pull")
-        downloaded_items_num = self.repo.cache.local.pull(
+
+        return self.repo.cache.local.pull(
             cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
-        if not remote.verify:
-            self._save_pulled_checksums(cache)
-
-        return downloaded_items_num
-
-    def _save_pulled_checksums(self, cache):
-        for checksum in cache["local"].keys():
-            cache_file = self.repo.cache.local.checksum_to_path_info(checksum)
-            if self.repo.cache.local.exists(cache_file):
-                # We can safely save here, as existing corrupted files will be
-                # removed upon status, while files corrupted during download
-                # will not be moved from tmp_file (see `RemoteBASE.download()`)
-                self.repo.state.save(cache_file, checksum)
-
-    def status(self, cache, jobs=None, remote=None, show_checksums=False):
+    def status(
+        self,
+        cache,
+        jobs=None,
+        remote=None,
+        show_checksums=False,
+        log_missing=True,
+    ):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
             cache (NamedCache): named checksums to check status for.
             jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.RemoteBASE): optional remote to compare
+            remote (dvc.remote.base.BaseRemote): optional remote to compare
                 cache to. By default remote from core.remote config option
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
+            log_missing (bool): log warning messages if file doesn't exist
+                neither in cache, neither in cloud.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache,
+            jobs=jobs,
+            remote=remote,
+            show_checksums=show_checksums,
+            log_missing=log_missing,
         )
+
+    def get_url_for(self, remote, checksum):
+        remote = self.get_remote(remote)
+        return str(remote.tree.hash_to_path_info(checksum))

dvc/dependency/__init__.py

@@ -1,37 +1,42 @@
+from collections import defaultdict
 from urllib.parse import urlparse
 
 import dvc.output as output
-from dvc.dependency.gs import DependencyGS
-from dvc.dependency.hdfs import DependencyHDFS
-from dvc.dependency.http import DependencyHTTP
-from dvc.dependency.https import DependencyHTTPS
-from dvc.dependency.local import DependencyLOCAL
-from dvc.dependency.s3 import DependencyS3
-from dvc.dependency.ssh import DependencySSH
-from dvc.output.base import OutputBase
-from dvc.remote import Remote
+from dvc.dependency.azure import AzureDependency
+from dvc.dependency.gs import GSDependency
+from dvc.dependency.hdfs import HDFSDependency
+from dvc.dependency.http import HTTPDependency
+from dvc.dependency.https import HTTPSDependency
+from dvc.dependency.local import LocalDependency
+from dvc.dependency.param import ParamsDependency
+from dvc.dependency.s3 import S3Dependency
+from dvc.dependency.ssh import SSHDependency
+from dvc.output.base import BaseOutput
 from dvc.scheme import Schemes
-from .repo import DependencyREPO
 
+from ..tree import get_cloud_tree
+from .repo import RepoDependency
 
 DEPS = [
-    DependencyGS,
-    DependencyHDFS,
-    DependencyHTTP,
-    DependencyHTTPS,
-    DependencyS3,
-    DependencySSH,
-    # NOTE: DependencyLOCAL is the default choice
+    AzureDependency,
+    GSDependency,
+    HDFSDependency,
+    HTTPDependency,
+    HTTPSDependency,
+    S3Dependency,
+    SSHDependency,
+    # NOTE: LocalDependency is the default choice
 ]
 
 DEP_MAP = {
-    Schemes.LOCAL: DependencyLOCAL,
-    Schemes.SSH: DependencySSH,
-    Schemes.S3: DependencyS3,
-    Schemes.GS: DependencyGS,
-    Schemes.HDFS: DependencyHDFS,
-    Schemes.HTTP: DependencyHTTP,
-    Schemes.HTTPS: DependencyHTTPS,
+    Schemes.LOCAL: LocalDependency,
+    Schemes.SSH: SSHDependency,
+    Schemes.S3: S3Dependency,
+    Schemes.AZURE: AzureDependency,
+    Schemes.GS: GSDependency,
+    Schemes.HDFS: HDFSDependency,
+    Schemes.HTTP: HTTPDependency,
+    Schemes.HTTPS: HTTPSDependency,
 }
 
 
@@ -40,39 +45,69 @@ DEP_MAP = {
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
 # output is a metric file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
-del SCHEMA[OutputBase.PARAM_CACHE]
-del SCHEMA[OutputBase.PARAM_METRIC]
-SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA
+del SCHEMA[BaseOutput.PARAM_CACHE]
+del SCHEMA[BaseOutput.PARAM_METRIC]
+SCHEMA.update(RepoDependency.REPO_SCHEMA)
+SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 
 
 def _get(stage, p, info):
-    parsed = urlparse(p)
+    parsed = urlparse(p) if p else None
+    if parsed and parsed.scheme == "remote":
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
 
-    if parsed.scheme == "remote":
-        remote = Remote(stage.repo, name=parsed.netloc)
-        return DEP_MAP[remote.scheme](stage, p, info, remote=remote)
+    if info and info.get(RepoDependency.PARAM_REPO):
+        repo = info.pop(RepoDependency.PARAM_REPO)
+        return RepoDependency(repo, stage, p, info)
 
-    if info and info.get(DependencyREPO.PARAM_REPO):
-        repo = info.pop(DependencyREPO.PARAM_REPO)
-        return DependencyREPO(repo, stage, p, info)
+    if info and info.get(ParamsDependency.PARAM_PARAMS):
+        params = info.pop(ParamsDependency.PARAM_PARAMS)
+        return ParamsDependency(stage, p, params)
 
     for d in DEPS:
         if d.supported(p):
             return d(stage, p, info)
-    return DependencyLOCAL(stage, p, info)
+    return LocalDependency(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(OutputBase.PARAM_PATH)
+        p = d.pop(BaseOutput.PARAM_PATH, None)
         ret.append(_get(stage, p, d))
     return ret
 
 
 def loads_from(stage, s_list, erepo=None):
-    ret = []
-    for s in s_list:
-        info = {DependencyREPO.PARAM_REPO: erepo} if erepo else {}
-        ret.append(_get(stage, s, info))
-    return ret
+    assert isinstance(s_list, list)
+    info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
+    return [_get(stage, s, info.copy()) for s in s_list]
+
+
+def _merge_params(s_list):
+    d = defaultdict(list)
+    default_file = ParamsDependency.DEFAULT_PARAMS_FILE
+    for key in s_list:
+        if isinstance(key, str):
+            d[default_file].append(key)
+            continue
+        if not isinstance(key, dict):
+            msg = "Only list of str/dict is supported. Got: "
+            msg += f"'{type(key).__name__}'."
+            raise ValueError(msg)
+
+        for k, params in key.items():
+            if not isinstance(params, list):
+                msg = "Expected list of params for custom params file "
+                msg += f"'{k}', got '{type(params).__name__}'."
+                raise ValueError(msg)
+            d[k].extend(params)
+    return d
+
+
+def loads_params(stage, s_list):
+    d = _merge_params(s_list)
+    return [
+        ParamsDependency(stage, path, params) for path, params in d.items()
+    ]

dvc/dependency/azure.py

@@ -0,0 +1,8 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+
+from ..tree.azure import AzureTree
+
+
+class AzureDependency(BaseDependency, BaseOutput):
+    TREE_CLS = AzureTree

dvc/dependency/base.py

@@ -3,24 +3,22 @@ from dvc.exceptions import DvcException
 
 class DependencyDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = "dependency '{}' does not exist".format(path)
+        msg = f"dependency '{path}' does not exist"
         super().__init__(msg)
 
 
 class DependencyIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = "dependency '{}' is not a file or directory".format(path)
+        msg = f"dependency '{path}' is not a file or directory"
         super().__init__(msg)
 
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__(
-            "Stage file '{}' cannot be a dependency.".format(path)
-        )
+        super().__init__(f"Stage file '{path}' cannot be a dependency.")
 
 
-class DependencyBase(object):
+class BaseDependency:
     IS_DEPENDENCY = True
 
     DoesNotExistError = DependencyDoesNotExistError

dvc/dependency/gs.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.gs import OutputGS
+from dvc.dependency.base import BaseDependency
+from dvc.output.gs import GSOutput
 
 
-class DependencyGS(DependencyBase, OutputGS):
+class GSDependency(BaseDependency, GSOutput):
     pass

dvc/dependency/hdfs.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.hdfs import OutputHDFS
+from dvc.dependency.base import BaseDependency
+from dvc.output.hdfs import HDFSOutput
 
 
-class DependencyHDFS(DependencyBase, OutputHDFS):
+class HDFSDependency(BaseDependency, HDFSOutput):
     pass

dvc/dependency/http.py

@@ -1,7 +1,8 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.base import OutputBase
-from dvc.remote.http import RemoteHTTP
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+
+from ..tree.http import HTTPTree
 
 
-class DependencyHTTP(DependencyBase, OutputBase):
-    REMOTE = RemoteHTTP
+class HTTPDependency(BaseDependency, BaseOutput):
+    TREE_CLS = HTTPTree

dvc/dependency/https.py

@@ -1,6 +1,6 @@
-from .http import DependencyHTTP
-from dvc.remote.https import RemoteHTTPS
+from ..tree.https import HTTPSTree
+from .http import HTTPDependency
 
 
-class DependencyHTTPS(DependencyHTTP):
-    REMOTE = RemoteHTTPS
+class HTTPSDependency(HTTPDependency):
+    TREE_CLS = HTTPSTree

dvc/dependency/local.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.local import OutputLOCAL
+from dvc.dependency.base import BaseDependency
+from dvc.output.local import LocalOutput
 
 
-class DependencyLOCAL(DependencyBase, OutputLOCAL):
+class LocalDependency(BaseDependency, LocalOutput):
     pass

dvc/dependency/param.py

@@ -0,0 +1,119 @@
+import os
+from collections import defaultdict
+
+import dpath.util
+from voluptuous import Any
+
+from dvc.dependency.local import LocalDependency
+from dvc.exceptions import DvcException
+from dvc.hash_info import HashInfo
+from dvc.utils.serialize import LOADERS, ParseError
+
+
+class MissingParamsError(DvcException):
+    pass
+
+
+class BadParamFileError(DvcException):
+    pass
+
+
+class ParamsDependency(LocalDependency):
+    PARAM_PARAMS = "params"
+    PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
+    DEFAULT_PARAMS_FILE = "params.yaml"
+
+    def __init__(self, stage, path, params):
+        info = {}
+        self.params = []
+        if params:
+            if isinstance(params, list):
+                self.params = params
+            else:
+                assert isinstance(params, dict)
+                self.params = list(params.keys())
+                info = {self.PARAM_PARAMS: params}
+
+        super().__init__(
+            stage,
+            path
+            or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
+            info=info,
+        )
+
+    def dumpd(self):
+        ret = super().dumpd()
+        if not self.hash_info:
+            ret[self.PARAM_PARAMS] = self.params
+        return ret
+
+    def fill_values(self, values=None):
+        """Load params values dynamically."""
+        if not values:
+            return
+        info = {}
+        for param in self.params:
+            if param in values:
+                info[param] = values[param]
+        self.hash_info = HashInfo(self.PARAM_PARAMS, info)
+
+    def workspace_status(self):
+        status = super().workspace_status()
+
+        if status.get(str(self)) == "deleted":
+            return status
+
+        status = defaultdict(dict)
+        info = self.hash_info.value if self.hash_info else {}
+        actual = self.read_params()
+        for param in self.params:
+            if param not in actual.keys():
+                st = "deleted"
+            elif param not in info:
+                st = "new"
+            elif actual[param] != info[param]:
+                st = "modified"
+            else:
+                assert actual[param] == info[param]
+                continue
+
+            status[str(self)][param] = st
+
+        return status
+
+    def status(self):
+        return self.workspace_status()
+
+    def read_params(self):
+        if not self.exists:
+            return {}
+
+        suffix = self.path_info.suffix.lower()
+        loader = LOADERS[suffix]
+        try:
+            config = loader(self.path_info, tree=self.repo.tree)
+        except ParseError as exc:
+            raise BadParamFileError(
+                f"Unable to read parameters from '{self}'"
+            ) from exc
+
+        ret = {}
+        for param in self.params:
+            try:
+                ret[param] = dpath.util.get(config, param, separator=".")
+            except KeyError:
+                pass
+        return ret
+
+    def get_hash(self):
+        info = self.read_params()
+
+        missing_params = set(self.params) - set(info.keys())
+        if missing_params:
+            raise MissingParamsError(
+                "Parameters '{}' are missing from '{}'.".format(
+                    ", ".join(missing_params), self,
+                )
+            )
+
+        return HashInfo(self.PARAM_PARAMS, info)

dvc/dependency/repo.py

@@ -1,29 +1,27 @@
-import os
-
 from voluptuous import Required
 
-from .local import DependencyLOCAL
-from dvc.exceptions import OutputNotFoundError
-from dvc.path_info import PathInfo
+from .local import LocalDependency
 
 
-class DependencyREPO(DependencyLOCAL):
+class RepoDependency(LocalDependency):
     PARAM_REPO = "repo"
     PARAM_URL = "url"
     PARAM_REV = "rev"
     PARAM_REV_LOCK = "rev_lock"
 
     REPO_SCHEMA = {
-        Required(PARAM_URL): str,
-        PARAM_REV: str,
-        PARAM_REV_LOCK: str,
+        PARAM_REPO: {
+            Required(PARAM_URL): str,
+            PARAM_REV: str,
+            PARAM_REV_LOCK: str,
+        }
     }
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, remote, path):
+    def _parse_path(self, tree, path):
         return None
 
     @property
@@ -39,31 +37,31 @@ class DependencyREPO(DependencyLOCAL):
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    def _make_repo(self, *, locked=True):
+    def _make_repo(self, *, locked=True, **kwargs):
         from dvc.external_repo import external_repo
 
         d = self.def_repo
         rev = (d.get("rev_lock") if locked else None) or d.get("rev")
-        return external_repo(d["url"], rev=rev)
+        return external_repo(d["url"], rev=rev, **kwargs)
 
-    def _get_checksum(self, locked=True):
-        with self._make_repo(locked=locked) as repo:
-            try:
-                return repo.find_out_by_relpath(self.def_path).info["md5"]
-            except OutputNotFoundError:
-                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
-                # We are polluting our repo cache with some dir listing here
-                return self.repo.cache.local.get_checksum(path)
+    def _get_hash(self, locked=True):
+        # we want stream but not fetch, so DVC out directories are
+        # walked, but dir contents is not fetched
+        with self._make_repo(locked=locked, fetch=False, stream=True) as repo:
+            return repo.get_checksum(self.def_path)
 
-    def status(self):
-        current_checksum = self._get_checksum(locked=True)
-        updated_checksum = self._get_checksum(locked=False)
+    def workspace_status(self):
+        current = self._get_hash(locked=True)
+        updated = self._get_hash(locked=False)
 
-        if current_checksum != updated_checksum:
+        if current != updated:
             return {str(self): "update available"}
 
         return {}
 
+    def status(self):
+        return self.workspace_status()
+
     def save(self):
         pass
 
@@ -71,18 +69,25 @@ class DependencyREPO(DependencyLOCAL):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
     def download(self, to):
-        with self._make_repo() as repo:
+        cache = self.repo.cache.local
+
+        with self._make_repo(cache_dir=cache.cache_dir) as repo:
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+                self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
-            if hasattr(repo, "cache"):
-                repo.cache.local.cache_dir = self.repo.cache.local.cache_dir
+            _, _, cache_infos = repo.fetch_external([self.def_path])
 
-            repo.pull_to(self.def_path, to.path_info)
+        cache.checkout(to.path_info, cache_infos[0])
 
     def update(self, rev=None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
 
         with self._make_repo(locked=False) as repo:
-            self.def_repo[self.PARAM_REV_LOCK] = repo.scm.get_rev()
+            self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
+
+    def changed_checksum(self):
+        # From current repo point of view what describes RepoDependency is its
+        # origin project url and rev_lock, and it makes RepoDependency
+        # immutable, hence its impossible for checksum to change.
+        return False

dvc/dependency/s3.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.s3 import OutputS3
+from dvc.dependency.base import BaseDependency
+from dvc.output.s3 import S3Output
 
 
-class DependencyS3(DependencyBase, OutputS3):
+class S3Dependency(BaseDependency, S3Output):
     pass

dvc/dependency/ssh.py

@@ -1,6 +1,6 @@
-from dvc.dependency.base import DependencyBase
-from dvc.output.ssh import OutputSSH
+from dvc.dependency.base import BaseDependency
+from dvc.output.ssh import SSHOutput
 
 
-class DependencySSH(DependencyBase, OutputSSH):
+class SSHDependency(BaseDependency, SSHOutput):
     pass

dvc/dvcfile.py

@@ -0,0 +1,329 @@
+import collections
+import contextlib
+import logging
+import os
+
+from voluptuous import MultipleInvalid
+
+from dvc.exceptions import DvcException
+from dvc.stage import serialize
+from dvc.stage.exceptions import (
+    StageFileBadNameError,
+    StageFileDoesNotExistError,
+    StageFileFormatError,
+    StageFileIsNotDvcFileError,
+)
+from dvc.stage.loader import SingleStageLoader, StageLoader
+from dvc.utils import relpath
+from dvc.utils.collections import apply_diff
+from dvc.utils.serialize import (
+    dump_yaml,
+    load_yaml,
+    modify_yaml,
+    parse_yaml,
+    parse_yaml_for_update,
+)
+
+logger = logging.getLogger(__name__)
+
+DVC_FILE = "Dvcfile"
+DVC_FILE_SUFFIX = ".dvc"
+PIPELINE_FILE = "dvc.yaml"
+PIPELINE_LOCK = "dvc.lock"
+
+
+class LockfileCorruptedError(DvcException):
+    pass
+
+
+def is_valid_filename(path):
+    return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
+        DVC_FILE,
+        PIPELINE_FILE,
+    ]
+
+
+def is_dvc_file(path):
+    return os.path.isfile(path) and (
+        is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
+    )
+
+
+def check_dvc_filename(path):
+    if not is_valid_filename(path):
+        raise StageFileBadNameError(
+            "bad DVC-file name '{}'. DVC-files should be named "
+            "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
+                relpath(path), os.path.basename(path)
+            )
+        )
+
+
+class FileMixin:
+    SCHEMA = None
+
+    def __init__(self, repo, path, verify=True, **kwargs):
+        self.repo = repo
+        self.path = path
+        self.verify = verify
+
+    def __repr__(self):
+        return "{}: {}".format(
+            self.__class__.__name__, relpath(self.path, self.repo.root_dir)
+        )
+
+    def __hash__(self):
+        return hash(self.path)
+
+    def __eq__(self, other):
+        return self.repo == other.repo and os.path.abspath(
+            self.path
+        ) == os.path.abspath(other.path)
+
+    def __str__(self):
+        return f"{self.__class__.__name__}: {self.relpath}"
+
+    @property
+    def relpath(self):
+        return relpath(self.path)
+
+    def exists(self):
+        return self.repo.tree.exists(self.path)
+
+    def _load(self):
+        # it raises the proper exceptions by priority:
+        # 1. when the file doesn't exists
+        # 2. filename is not a DVC-file
+        # 3. path doesn't represent a regular file
+        if not self.exists():
+            raise StageFileDoesNotExistError(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
+        if not self.repo.tree.isfile(self.path):
+            raise StageFileIsNotDvcFileError(self.path)
+
+        with self.repo.tree.open(self.path) as fd:
+            stage_text = fd.read()
+        d = parse_yaml(stage_text, self.path)
+        self.validate(d, self.relpath)
+        return d, stage_text
+
+    @classmethod
+    def validate(cls, d, fname=None):
+        assert isinstance(cls.SCHEMA, collections.abc.Callable)
+        try:
+            cls.SCHEMA(d)  # pylint: disable=not-callable
+        except MultipleInvalid as exc:
+            raise StageFileFormatError(f"'{fname}' format error: {exc}")
+
+    def remove(self, force=False):  # pylint: disable=unused-argument
+        with contextlib.suppress(FileNotFoundError):
+            os.unlink(self.path)
+
+    def dump(self, stage, **kwargs):
+        raise NotImplementedError
+
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
+
+class SingleStageFile(FileMixin):
+    from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
+
+    @property
+    def stage(self):
+        data, raw = self._load()
+        return SingleStageLoader.load_stage(self, data, raw)
+
+    @property
+    def stages(self):
+        data, raw = self._load()
+        return SingleStageLoader(self, data, raw)
+
+    def dump(self, stage, **kwargs):
+        """Dumps given stage appropriately in the dvcfile."""
+        from dvc.stage import PipelineStage
+
+        assert not isinstance(stage, PipelineStage)
+        if self.verify:
+            check_dvc_filename(self.path)
+        logger.debug(
+            "Saving information to '{file}'.".format(file=relpath(self.path))
+        )
+        dump_yaml(self.path, serialize.to_single_stage_file(stage))
+        self.repo.scm.track_file(self.relpath)
+
+    def remove_stage(self, stage):  # pylint: disable=unused-argument
+        self.remove()
+
+    def merge(self, ancestor, other):
+        assert isinstance(ancestor, SingleStageFile)
+        assert isinstance(other, SingleStageFile)
+
+        stage = self.stage
+        stage.merge(ancestor.stage, other.stage)
+        self.dump(stage)
+
+
+class PipelineFile(FileMixin):
+    """Abstraction for pipelines file, .yaml + .lock combined."""
+
+    from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
+
+    @property
+    def _lockfile(self):
+        return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
+
+    def dump(
+        self, stage, update_pipeline=True, update_lock=True, **kwargs
+    ):  # pylint: disable=arguments-differ
+        """Dumps given stage appropriately in the dvcfile."""
+        from dvc.stage import PipelineStage
+
+        assert isinstance(stage, PipelineStage)
+        if self.verify:
+            check_dvc_filename(self.path)
+
+        if update_pipeline and not stage.is_data_source:
+            self._dump_pipeline_file(stage)
+
+        if update_lock:
+            self._dump_lockfile(stage)
+
+    def _dump_lockfile(self, stage):
+        self._lockfile.dump(stage)
+
+    def _dump_pipeline_file(self, stage):
+        stage_data = serialize.to_pipeline_file(stage)
+
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
+            if not data:
+                logger.info("Creating '%s'", self.relpath)
+
+            data["stages"] = data.get("stages", {})
+            existing_entry = stage.name in data["stages"]
+            action = "Modifying" if existing_entry else "Adding"
+            logger.info(
+                "%s stage '%s' in '%s'", action, stage.name, self.relpath
+            )
+
+            if existing_entry:
+                orig_stage_data = data["stages"][stage.name]
+                if "meta" in orig_stage_data:
+                    stage_data[stage.name]["meta"] = orig_stage_data["meta"]
+                apply_diff(stage_data[stage.name], orig_stage_data)
+            else:
+                data["stages"].update(stage_data)
+
+        self.repo.scm.track_file(self.relpath)
+
+    @property
+    def stage(self):
+        raise DvcException(
+            "PipelineFile has multiple stages. Please specify it's name."
+        )
+
+    @property
+    def stages(self):
+        data, _ = self._load()
+        lockfile_data = self._lockfile.load()
+        return StageLoader(self, data.get("stages", {}), lockfile_data)
+
+    def remove(self, force=False):
+        if not force:
+            logger.warning("Cannot remove pipeline file.")
+            return
+
+        super().remove()
+        self._lockfile.remove()
+
+    def remove_stage(self, stage):
+        self._lockfile.remove_stage(stage)
+        if not self.exists():
+            return
+
+        with open(self.path, "r") as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+
+        self.validate(d, self.path)
+        if stage.name not in d.get("stages", {}):
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d["stages"][stage.name]
+
+        if d["stages"]:
+            dump_yaml(self.path, d)
+        else:
+            super().remove()
+
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
+
+class Lockfile(FileMixin):
+    from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
+
+    def load(self):
+        if not self.exists():
+            return {}
+
+        data = load_yaml(self.path, tree=self.repo.tree)
+        try:
+            self.validate(data, fname=self.relpath)
+        except StageFileFormatError:
+            raise LockfileCorruptedError(
+                f"Lockfile '{self.relpath}' is corrupted."
+            )
+        return data
+
+    def dump(self, stage, **kwargs):
+        stage_data = serialize.to_lockfile(stage)
+
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
+            if not data:
+                logger.info("Generating lock file '%s'", self.relpath)
+
+            modified = data.get(stage.name, {}) != stage_data.get(
+                stage.name, {}
+            )
+            if modified:
+                logger.info("Updating lock file '%s'", self.relpath)
+            data.update(stage_data)
+
+        if modified:
+            self.repo.scm.track_file(self.relpath)
+
+    def remove_stage(self, stage):
+        if not self.exists():
+            return
+
+        with open(self.path) as f:
+            d = parse_yaml_for_update(f.read(), self.path)
+        self.validate(d, self.path)
+
+        if stage.name not in d:
+            return
+
+        logger.debug("Removing '%s' from '%s'", stage.name, self.path)
+        del d[stage.name]
+
+        if d:
+            dump_yaml(self.path, d)
+        else:
+            self.remove()
+
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
+
+class Dvcfile:
+    def __new__(cls, repo, path, **kwargs):
+        assert path
+        assert repo
+
+        _, ext = os.path.splitext(path)
+        if ext in [".yaml", ".yml"]:
+            return PipelineFile(repo, path, **kwargs)
+        # fallback to single stage file for better error messages
+        return SingleStageFile(repo, path, **kwargs)

dvc/exceptions.py

@@ -1,11 +1,20 @@
 """Exceptions raised by the dvc."""
+from funcy import first
 
-from dvc.utils import relpath, format_link
+from dvc.utils import error_link, format_link, relpath
 
 
 class DvcException(Exception):
     """Base class for all dvc exceptions."""
 
+    def __init__(self, msg, *args):
+        assert msg
+        super().__init__(msg, *args)
+
+
+class InvalidArgumentError(ValueError, DvcException):
+    """Thrown if arguments are invalid."""
+
 
 class OutputDuplicationError(DvcException):
     """Thrown if a file/directory is specified as an output in more than one
@@ -19,12 +28,17 @@ class OutputDuplicationError(DvcException):
     def __init__(self, output, stages):
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
-        msg = (
-            "file/directory '{}' is specified as an output in more than one "
-            "stage: {}\n"
-            "This is not allowed. Consider using a different output name."
-        ).format(output, "\n    ".join(s.relpath for s in stages))
+        if len(stages) == 1:
+            msg = "output '{}' is already specified in {}.".format(
+                output, first(stages)
+            )
+        else:
+            msg = "output '{}' is already specified in stages:\n{}".format(
+                output, "\n".join(f"\t- {s.addressing}" for s in stages),
+            )
         super().__init__(msg)
+        self.stages = stages
+        self.output = output
 
 
 class OutputNotFoundError(DvcException):
@@ -56,8 +70,8 @@ class StagePathAsOutputError(DvcException):
     def __init__(self, stage, output):
         assert isinstance(output, str)
         super().__init__(
-            "'{stage}' is within an output '{output}' of another stage".format(
-                stage=stage.relpath, output=output
+            "{stage} is within an output '{output}' of another stage".format(
+                stage=stage, output=output
             )
         )
 
@@ -73,10 +87,7 @@ class CircularDependencyError(DvcException):
     def __init__(self, dependency):
         assert isinstance(dependency, str)
 
-        msg = (
-            "file/directory '{}' is specified as an output and as a "
-            "dependency."
-        )
+        msg = "'{}' is specified as an output and as a dependency."
         super().__init__(msg.format(dependency))
 
 
@@ -90,7 +101,7 @@ class ArgumentDuplicationError(DvcException):
 
     def __init__(self, path):
         assert isinstance(path, str)
-        super().__init__("file '{}' is specified more than once.".format(path))
+        super().__init__(f"file '{path}' is specified more than once.")
 
 
 class MoveNotDataSourceError(DvcException):
@@ -125,12 +136,8 @@ class DvcParserError(DvcException):
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
-        stages = "\n".join("\t- {}".format(stage.relpath) for stage in stages)
-        msg = (
-            "you've introduced a cycle in your pipeline that involves "
-            "the following stages:"
-            "\n"
-            "{stages}".format(stages=stages)
+        msg = "Pipeline has a cycle involving: {}.".format(
+            ", ".join(s.addressing for s in stages)
         )
         super().__init__(msg)
 
@@ -138,7 +145,7 @@ class CyclicGraphError(DvcException):
 class ConfirmRemoveError(DvcException):
     def __init__(self, path):
         super().__init__(
-            "unable to remove '{}' without a confirmation from the user. Use "
+            "unable to remove '{}' without a confirmation. Use "
             "`-f` to force.".format(path)
         )
 
@@ -150,7 +157,7 @@ class InitError(DvcException):
 class ReproductionError(DvcException):
     def __init__(self, dvc_file_name):
         self.path = dvc_file_name
-        super().__init__("failed to reproduce '{}'".format(dvc_file_name))
+        super().__init__(f"failed to reproduce '{dvc_file_name}'")
 
 
 class BadMetricError(DvcException):
@@ -158,25 +165,20 @@ class BadMetricError(DvcException):
         super().__init__(
             "the following metrics do not exist, "
             "are not metric files or are malformed: {paths}".format(
-                paths=", ".join("'{}'".format(path) for path in paths)
+                paths=", ".join(f"'{path}'" for path in paths)
             )
         )
 
 
 class NoMetricsError(DvcException):
+    pass
+
+
+class NoPlotsError(DvcException):
     def __init__(self):
         super().__init__(
-            "no metric files in this repository. "
-            "Use `dvc metrics add` to add a metric file to track."
-        )
-
-
-class StageFileCorruptedError(DvcException):
-    def __init__(self, path):
-        path = relpath(path)
-        super().__init__(
-            "unable to read DVC-file: {} "
-            "YAML file structure is corrupted".format(path)
+            "no plots in this repository. Use `--plots/--plots-no-cache` "
+            "options for `dvc run` to mark stage outputs as plots."
         )
 
 
@@ -196,7 +198,7 @@ class OverlappingOutputPathsError(DvcException):
 
 class CheckoutErrorSuggestGit(DvcException):
     def __init__(self, target):
-        super().__init__("Did you mean `git checkout {}`?".format(target))
+        super().__init__(f"Did you mean `git checkout {target}`?")
 
 
 class ETagMismatchError(DvcException):
@@ -208,10 +210,11 @@ class ETagMismatchError(DvcException):
 
 
 class FileMissingError(DvcException):
-    def __init__(self, path):
+    def __init__(self, path, hint=None):
         self.path = path
+        hint = "" if hint is None else f". {hint}"
         super().__init__(
-            "Can't find '{}' neither locally nor on remote".format(path)
+            f"Can't find '{path}' neither locally nor on remote{hint}"
         )
 
 
@@ -237,26 +240,26 @@ class DownloadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(
-            "{amount} files failed to download".format(amount=amount)
-        )
+        super().__init__(f"{amount} files failed to download")
 
 
 class UploadError(DvcException):
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(
-            "{amount} files failed to upload".format(amount=amount)
-        )
+        super().__init__(f"{amount} files failed to upload")
 
 
 class CheckoutError(DvcException):
-    def __init__(self, target_infos):
+    def __init__(self, target_infos, stats=None):
+        self.target_infos = target_infos
+        self.stats = stats
         targets = [str(t) for t in target_infos]
         m = (
-            "Checkout failed for following targets:\n {}\nDid you "
-            "forget to fetch?".format("\n".join(targets))
+            "Checkout failed for following targets:\n{}\nIs your "
+            "cache up to date?\n{}".format(
+                "\n".join(targets), error_link("missing-files"),
+            )
         )
         super().__init__(m)
 
@@ -268,7 +271,7 @@ class CollectCacheError(DvcException):
 class NoRemoteInExternalRepoError(DvcException):
     def __init__(self, url):
         super().__init__(
-            "No DVC remote is specified in target repository '{}'.".format(url)
+            f"No DVC remote is specified in target repository '{url}'."
         )
 
 
@@ -283,27 +286,23 @@ class NoOutputInExternalRepoError(DvcException):
 
 class HTTPError(DvcException):
     def __init__(self, code, reason):
-        super().__init__("'{} {}'".format(code, reason))
+        super().__init__(f"'{code} {reason}'")
 
 
 class PathMissingError(DvcException):
     default_msg = (
         "The path '{}' does not exist in the target repository '{}'"
-        " neither as an output nor a git-handled file."
+        " neither as a DVC output nor as a Git-tracked file."
     )
-    default_msg_output_only = (
+    default_msg_dvc_only = (
         "The path '{}' does not exist in the target repository '{}'"
-        " as an output."
+        " as an DVC output."
     )
 
-    def __init__(self, path, repo, output_only=False):
-        msg = (
-            self.default_msg
-            if not output_only
-            else self.default_msg_output_only
-        )
+    def __init__(self, path, repo, dvc_only=False):
+        msg = self.default_msg if not dvc_only else self.default_msg_dvc_only
         super().__init__(msg.format(path, repo))
-        self.output_only = output_only
+        self.dvc_only = dvc_only
 
 
 class RemoteCacheRequiredError(DvcException):
@@ -319,3 +318,23 @@ class RemoteCacheRequiredError(DvcException):
                 format_link("https://man.dvc.org/config#cache"),
             )
         )
+
+
+class IsADirectoryError(DvcException):  # noqa,pylint:disable=redefined-builtin
+    """Raised when a file operation is requested on a directory."""
+
+
+class NoOutputOrStageError(DvcException):
+    """
+    Raised when the target is neither an output nor a stage name in dvc.yaml
+    """
+
+    def __init__(self, target, file):
+        super().__init__(
+            f"'{target}' "
+            f"does not exist as an output or a stage name in '{file}'"
+        )
+
+
+class MergeError(DvcException):
+    pass

dvc/external_repo.py

@@ -1,46 +1,75 @@
 import logging
 import os
 import tempfile
+import threading
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
-import threading
+from typing import Iterable
 
-from funcy import retry, suppress, wrap_with, cached_property
+from funcy import cached_property, reraise, retry, wrap_with
 
-from dvc.path_info import PathInfo
-from dvc.compat import fspath
-from dvc.repo import Repo
+from dvc.cache import Cache
 from dvc.config import NoRemoteError, NotDvcRepoError
-from dvc.exceptions import NoRemoteInExternalRepoError
-from dvc.exceptions import OutputNotFoundError, NoOutputInExternalRepoError
-from dvc.exceptions import FileMissingError, PathMissingError
-from dvc.utils.fs import remove, fs_copy, move
-from dvc.utils import tmp_fname
+from dvc.exceptions import (
+    DvcException,
+    FileMissingError,
+    NoOutputInExternalRepoError,
+    NoRemoteInExternalRepoError,
+    OutputNotFoundError,
+    PathMissingError,
+)
+from dvc.path_info import PathInfo
+from dvc.repo import Repo
+from dvc.scm.base import CloneError
 from dvc.scm.git import Git
-
+from dvc.state import StateNoop
+from dvc.tree.local import LocalTree
+from dvc.tree.repo import RepoTree
+from dvc.utils import relpath
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
 
+class IsADVCRepoError(DvcException):
+    """Raised when it is not expected to be a dvc repo."""
+
+
 @contextmanager
-def external_repo(url, rev=None, for_write=False):
-    logger.debug("Creating external repo {}@{}", url, rev)
+def external_repo(url, rev=None, for_write=False, **kwargs):
+    logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
+    # Local HEAD points to the tip of whatever branch we first cloned from
+    # (which may not be the default branch), use origin/HEAD here to get
+    # the tip of the default branch
+    rev = rev or "refs/remotes/origin/HEAD"
+
+    root_dir = path if for_write else os.path.realpath(path)
+    conf = dict(
+        root_dir=root_dir,
+        url=url,
+        scm=None if for_write else Git(root_dir),
+        rev=None if for_write else rev,
+        for_write=for_write,
+        **kwargs,
+    )
     try:
-        repo = ExternalRepo(path, url)
+        repo = ExternalRepo(**conf)
     except NotDvcRepoError:
-        repo = ExternalGitRepo(path, url)
+        repo = ExternalGitRepo(**conf)
 
     try:
         yield repo
-    except NoRemoteError:
-        raise NoRemoteInExternalRepoError(url)
+    except NoRemoteError as exc:
+        raise NoRemoteInExternalRepoError(url) from exc
     except OutputNotFoundError as exc:
         if exc.repo is repo:
-            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
+            raise NoOutputInExternalRepoError(
+                exc.output, repo.root_dir, url
+            ) from exc
         raise
     except FileMissingError as exc:
-        raise PathMissingError(exc.path, url)
+        raise PathMissingError(exc.path, url) from exc
     finally:
         repo.close()
         if for_write:
@@ -53,7 +82,7 @@ CACHE_DIRS = {}
 
 def clean_repos():
     # Outside code should not see cache while we are removing
-    paths = list(CLONES.values()) + list(CACHE_DIRS.values())
+    paths = [path for path, _ in CLONES.values()] + list(CACHE_DIRS.values())
     CLONES.clear()
     CACHE_DIRS.clear()
 
@@ -61,119 +90,256 @@ def clean_repos():
         _remove(path)
 
 
-class ExternalRepo(Repo):
-    def __init__(self, root_dir, url):
-        super().__init__(root_dir)
-        self.url = url
-        self._set_cache_dir()
-        self._set_upstream()
+class BaseExternalRepo:
+    # pylint: disable=no-member
 
-    def pull_to(self, path, to_info):
-        """
-        Pull the corresponding file or directory specified by `path` and
-        checkout it into `to_info`.
+    def __str__(self):
+        return self.url
 
-        It works with files tracked by Git and DVC, and also local files
-        outside the repository.
+    @cached_property
+    def repo_tree(self):
+        return self._get_tree_for(
+            self, subrepos=not self.for_write, repo_factory=self.make_repo
+        )
+
+    def get_rev(self):
+        assert self.scm
+        if isinstance(self.tree, LocalTree):
+            return self.scm.get_rev()
+        return self.tree.rev
+
+    def _fetch_to_cache(self, path_info, repo, callback):
+        # don't support subrepo traversal as it might fail due to difference
+        # in remotes
+        tree = self._get_tree_for(repo)
+        cache = repo.cache.local
+
+        hash_info = tree.get_hash(path_info, download_callback=callback)
+        cache.save(
+            path_info,
+            tree,
+            hash_info,
+            save_link=False,
+            download_callback=callback,
+        )
+        return hash_info
+
+    def fetch_external(self, paths: Iterable, **kwargs):
+        """Fetch specified external repo paths into cache.
+
+        Returns 3-tuple in the form
+            (downloaded, failed, list(cache_infos))
+        where cache_infos can be used as checkout targets for the
+        fetched paths.
         """
-        out = None
+        download_results = []
+        failed = 0
+        root = PathInfo(self.root_dir)
+
+        paths = [root / path for path in paths]
+
+        def download_update(result):
+            download_results.append(result)
+
+        hash_infos = []
+        for path in paths:
+            with reraise(FileNotFoundError, PathMissingError(path, self.url)):
+                metadata = self.repo_tree.metadata(path)
+
+            self._check_repo(path, metadata.repo)
+            repo = metadata.repo
+            hash_info = self._fetch_to_cache(path, repo, download_update)
+            hash_infos.append(hash_info)
+
+        return sum(download_results), failed, hash_infos
+
+    def _check_repo(self, path_info, repo):
+        if not repo:
+            return
+
+        repo_path = PathInfo(repo.root_dir)
+        if path_info == repo_path and isinstance(repo, Repo):
+            message = "Cannot fetch a complete DVC repository"
+            if repo.root_dir != self.root_dir:
+                rel = relpath(repo.root_dir, self.root_dir)
+                message += f" '{rel}'"
+            raise IsADVCRepoError(message)
+
+    def get_external(self, path, dest):
+        """Convenience wrapper for fetch_external and checkout."""
         path_info = PathInfo(self.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, self.url)):
+            metadata = self.repo_tree.metadata(path_info)
 
-        with suppress(OutputNotFoundError):
-            (out,) = self.find_outs_by_path(fspath(path_info), strict=False)
+        self._check_repo(path_info, metadata.repo)
+        if metadata.output_exists:
+            repo = metadata.repo
+            cache = repo.cache.local
+            # fetch DVC and git files to tmpdir cache, then checkout
+            save_info = self._fetch_to_cache(path_info, repo, None)
+            cache.checkout(PathInfo(dest), save_info)
+        else:
+            # git-only folder, just copy files directly to dest
+            tree = self._get_tree_for(metadata.repo)  # ignore subrepos
+            tree.copytree(path_info, dest)
+
+    def _get_tree_for(self, repo, **kwargs):
+        """
+        Provides a combined tree of a single repo with dvc + git/local tree.
+        """
+        kw = {**self.tree_confs, **kwargs}
+        if "fetch" not in kw:
+            kw["fetch"] = True
+        return RepoTree(repo, **kw)
+
+    def get_checksum(self, path):
+        path_info = PathInfo(self.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, self.url)):
+            metadata = self.repo_tree.metadata(path_info)
+
+        # skip subrepos to check for
+        tree = self._get_tree_for(metadata.repo)
+        return tree.get_hash(path_info)
+
+    @staticmethod
+    def _fix_local_remote(orig_repo, src_repo, remote_name):
+        # If a remote URL is relative to the source repo,
+        # it will have changed upon config load and made
+        # relative to this new repo. Restore the old one here.
+        new_remote = orig_repo.config["remote"][remote_name]
+        old_remote = src_repo.config["remote"][remote_name]
+        if new_remote["url"] != old_remote["url"]:
+            new_remote["url"] = old_remote["url"]
+
+    @staticmethod
+    def _add_upstream(orig_repo, src_repo):
+        # Fill the empty upstream entry with a new remote pointing to the
+        # original repo's cache location.
+        cache_dir = src_repo.cache.local.cache_dir
+        orig_repo.config["remote"]["auto-generated-upstream"] = {
+            "url": cache_dir
+        }
+        orig_repo.config["core"]["remote"] = "auto-generated-upstream"
+
+    def make_repo(self, path):
+        repo = Repo(path, scm=self.scm, rev=self.get_rev())
+
+        self._setup_cache(repo)
+        self._fix_upstream(repo)
+
+        return repo
+
+    def _setup_cache(self, repo):
+        repo.cache.local.cache_dir = self.cache_dir
+        if self.cache_types:
+            repo.cache.local.cache_types = self.cache_types
+
+    def _fix_upstream(self, repo):
+        if not os.path.isdir(self.url):
+            return
 
         try:
-            if out and out.use_cache:
-                self._pull_cached(out, path_info, to_info)
-                return
+            rel_path = os.path.relpath(repo.root_dir, self.root_dir)
+            src_repo = Repo(PathInfo(self.url) / rel_path)
+        except NotDvcRepoError:
+            return
 
-            # Check if it is handled by Git (it can't have an absolute path)
-            if os.path.isabs(path):
-                raise FileNotFoundError
-
-            fs_copy(fspath(path_info), fspath(to_info))
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
-
-    def _pull_cached(self, out, path_info, dest):
-        with self.state:
-            tmp = PathInfo(tmp_fname(dest))
-            src = tmp / path_info.relative_to(out.path_info)
-
-            out.path_info = tmp
-
-            # Only pull unless all needed cache is present
-            if out.changed_cache(filter_info=src):
-                self.cloud.pull(out.get_used_cache(filter_info=src))
-
-            failed = out.checkout(filter_info=src)
-
-            move(src, dest)
-            remove(tmp)
-
-            if failed:
-                raise FileNotFoundError
+        try:
+            remote_name = repo.config["core"].get("remote")
+            if remote_name:
+                self._fix_local_remote(repo, src_repo, remote_name)
+            else:
+                self._add_upstream(repo, src_repo)
+        finally:
+            src_repo.close()
 
     @wrap_with(threading.Lock())
-    def _set_cache_dir(self):
+    def _get_cache_dir(self):
         try:
             cache_dir = CACHE_DIRS[self.url]
         except KeyError:
             cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
-
-        self.cache.local.cache_dir = cache_dir
-
-    def _set_upstream(self):
-        # check if the URL is local and no default remote is present
-        # add default remote pointing to the original repo's cache location
-        if os.path.isdir(self.url):
-            if not self.config["core"].get("remote"):
-                src_repo = Repo(self.url)
-                try:
-                    cache_dir = src_repo.cache.local.cache_dir
-                finally:
-                    src_repo.close()
-
-                self.config["remote"]["auto-generated-upstream"] = {
-                    "url": cache_dir
-                }
-                self.config["core"]["remote"] = "auto-generated-upstream"
+        return cache_dir
 
 
-class ExternalGitRepo:
-    def __init__(self, root_dir, url):
-        self.root_dir = root_dir
+class ExternalRepo(BaseExternalRepo, Repo):
+    def __init__(
+        self,
+        root_dir,
+        url,
+        scm=None,
+        rev=None,
+        for_write=False,
+        cache_dir=None,
+        cache_types=None,
+        **kwargs,
+    ):
+        super().__init__(root_dir, scm=scm, rev=rev)
+
         self.url = url
+        self.for_write = for_write
+        self.cache_dir = cache_dir or self._get_cache_dir()
+        self.cache_types = cache_types
+
+        self._setup_cache(self)
+        self._fix_upstream(self)
+        self.tree_confs = kwargs
+
+
+class ExternalGitRepo(BaseExternalRepo):
+    def __init__(
+        self,
+        root_dir,
+        url,
+        scm=None,
+        rev=None,
+        for_write=False,
+        cache_dir=None,
+        cache_types=None,
+        **kwargs,
+    ):
+        self.root_dir = os.path.realpath(root_dir)
+        self.scm = scm
+
+        self.url = url
+        self.for_write = for_write
+        self.cache_dir = cache_dir or self._get_cache_dir()
+        self.cache_types = cache_types
+
+        self.rev = rev
+        self.tree_confs = kwargs
+
+        self.config = {"cache": {"dir": self.cache_dir}}
+        self.cache = Cache(self)
+        if cache_types:
+            self.cache.local.cache_types = cache_types
+
+        self.state = StateNoop()
 
     @cached_property
-    def scm(self):
-        return Git(self.root_dir)
+    def tree(self):
+        if self.scm:
+            return self.scm.get_tree(self.rev)
+        return LocalTree(self, {"url": self.root_dir})
 
     def close(self):
-        if "scm" in self.__dict__:
+        if self.scm:
             self.scm.close()
 
     def find_out_by_relpath(self, path):
         raise OutputNotFoundError(path, self)
 
-    def pull_to(self, path, to_info):
-        try:
-            # Git handled files can't have absolute path
-            if os.path.isabs(path):
-                raise FileNotFoundError
-
-            fs_copy(os.path.join(self.root_dir, path), fspath(to_info))
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
-
     @contextmanager
     def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
+        """Opens a specified resource as a file object."""
+        path_info = PathInfo(self.root_dir) / path
         try:
-            abs_path = os.path.join(self.root_dir, path)
-            with open(abs_path, mode, encoding=encoding) as fd:
-                yield fd
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
+            with self.repo_tree.open(
+                path_info, mode=mode, encoding=encoding, **kwargs
+            ) as fobj:
+                yield fobj
+        except FileNotFoundError as exc:
+            raise PathMissingError(path, self.url) from exc
 
 
 def _cached_clone(url, rev, for_write=False):
@@ -183,35 +349,33 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    if not for_write and Git.is_sha(rev) and (url, rev) in CLONES:
-        return CLONES[url, rev]
+    # even if we have already cloned this repo, we may need to
+    # fetch/fast-forward to get specified rev
+    clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
 
-    clone_path = _clone_default_branch(url, rev)
-    rev_sha = Git(clone_path).resolve_rev(rev or "HEAD")
-
-    if not for_write and (url, rev_sha) in CLONES:
-        return CLONES[url, rev_sha]
+    if not for_write and (url) in CLONES:
+        return CLONES[url][0]
 
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
+    logger.debug("erepo: making a copy of %s clone", url)
     copy_tree(clone_path, repo_path)
 
     # Check out the specified revision
-    if rev is not None:
+    if for_write:
         _git_checkout(repo_path, rev)
-
-    if not for_write:
-        CLONES[url, rev_sha] = repo_path
+    else:
+        CLONES[url] = (repo_path, shallow)
     return repo_path
 
 
 @wrap_with(threading.Lock())
-def _clone_default_branch(url, rev):
+def _clone_default_branch(url, rev, for_write=False):
     """Get or create a clean clone of the url.
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
-    clone_path = CLONES.get(url)
+    clone_path, shallow = CLONES.get(url, (None, False))
 
     git = None
     try:
@@ -219,19 +383,59 @@ def _clone_default_branch(url, rev):
             git = Git(clone_path)
             # Do not pull for known shas, branches and tags might move
             if not Git.is_sha(rev) or not git.has_rev(rev):
-                git.pull()
+                if shallow:
+                    # If we are missing a rev in a shallow clone, fallback to
+                    # a full (unshallowed) clone. Since fetching specific rev
+                    # SHAs is only available in certain git versions, if we
+                    # have need to reference multiple specific revs for a
+                    # given repo URL it is easier/safer for us to work with
+                    # full clones in this case.
+                    logger.debug("erepo: unshallowing clone for '%s'", url)
+                    _unshallow(git)
+                    shallow = False
+                    CLONES[url] = (clone_path, shallow)
+                else:
+                    logger.debug("erepo: git pull '%s'", url)
+                    git.pull()
         else:
+            logger.debug("erepo: git clone '%s' to a temporary dir", url)
             clone_path = tempfile.mkdtemp("dvc-clone")
-            git = Git.clone(url, clone_path)
-            CLONES[url] = clone_path
+            if not for_write and rev and not Git.is_sha(rev):
+                # If rev is a tag or branch name try shallow clone first
+                try:
+                    git = Git.clone(url, clone_path, shallow_branch=rev)
+                    shallow = True
+                    logger.debug(
+                        "erepo: using shallow clone for branch '%s'", rev
+                    )
+                except CloneError:
+                    pass
+            if not git:
+                git = Git.clone(url, clone_path)
+                shallow = False
+            CLONES[url] = (clone_path, shallow)
     finally:
         if git:
             git.close()
 
-    return clone_path
+    return clone_path, shallow
+
+
+def _unshallow(git):
+    if git.repo.head.is_detached:
+        # If this is a detached head (i.e. we shallow cloned a tag) switch to
+        # the default branch
+        origin_refs = git.repo.remotes["origin"].refs
+        ref = origin_refs["HEAD"].reference
+        branch_name = ref.name.split("/")[-1]
+        branch = git.repo.create_head(branch_name, ref)
+        branch.set_tracking_branch(ref)
+        branch.checkout()
+    git.pull(unshallow=True)
 
 
 def _git_checkout(repo_path, rev):
+    logger.debug("erepo: git checkout %s@%s", repo_path, rev)
     git = Git(repo_path)
     try:
         git.checkout(rev)
@@ -243,6 +447,11 @@ def _remove(path):
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(path)
+        try:
+            os_retry(remove)(path)
+        except PermissionError:
+            logger.warning(
+                "Failed to remove '%s'", relpath(path), exc_info=True
+            )
     else:
         remove(path)

dvc/hash_info.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass, field
+
+HASH_DIR_SUFFIX = ".dir"
+
+
+@dataclass
+class HashInfo:
+    name: str
+    value: str
+    dir_info: list = field(default=None, compare=False)
+
+    def __bool__(self):
+        return bool(self.value)
+
+    @classmethod
+    def from_dict(cls, d):
+        if not d:
+            return cls(None, None)
+        ((name, value),) = d.items()
+        return cls(name, value)
+
+    def to_dict(self):
+        return {self.name: self.value} if self else {}
+
+    @property
+    def isdir(self):
+        if not self:
+            return False
+        return self.value.endswith(HASH_DIR_SUFFIX)

dvc/ignore.py

@@ -1,17 +1,22 @@
 import logging
 import os
+import re
+from collections import namedtuple
+from itertools import groupby, takewhile
 
-from funcy import cached_property
-from pathspec import PathSpec
 from pathspec.patterns import GitWildMatchPattern
+from pathspec.util import normalize_file
 
-from dvc.scm.tree import BaseTree
+from dvc.path_info import PathInfo
+from dvc.pathspec_math import PatternInfo, merge_patterns
+from dvc.system import System
 from dvc.utils import relpath
+from dvc.utils.collections import PathStringTrie
 
 logger = logging.getLogger(__name__)
 
 
-class DvcIgnore(object):
+class DvcIgnore:
     DVCIGNORE_FILE = ".dvcignore"
 
     def __call__(self, root, dirs, files):
@@ -19,109 +24,332 @@ class DvcIgnore(object):
 
 
 class DvcIgnorePatterns(DvcIgnore):
-    def __init__(self, ignore_file_path, tree):
+    def __init__(self, pattern_list, dirname):
+        if pattern_list:
+            if isinstance(pattern_list[0], str):
+                pattern_list = [
+                    PatternInfo(pattern, "") for pattern in pattern_list
+                ]
+
+        self.pattern_list = pattern_list
+        self.dirname = dirname
+        self.prefix = self.dirname + os.sep
+
+        self.regex_pattern_list = [
+            GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
+            for pattern_info in pattern_list
+        ]
+
+        self.ignore_spec = [
+            (ignore, re.compile("|".join(item[0] for item in group)))
+            for ignore, group in groupby(
+                self.regex_pattern_list, lambda x: x[1]
+            )
+            if ignore is not None
+        ]
+
+    @classmethod
+    def from_files(cls, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
-
-        self.ignore_file_path = ignore_file_path
-        self.dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-
+        dirname = os.path.normpath(os.path.dirname(ignore_file_path))
+        ignore_file_rel_path = os.path.relpath(
+            ignore_file_path, tree.tree_root
+        )
         with tree.open(ignore_file_path, encoding="utf-8") as fobj:
-            self.ignore_spec = PathSpec.from_lines(GitWildMatchPattern, fobj)
+            path_spec_lines = [
+                PatternInfo(
+                    line,
+                    "{}:{}:{}".format(ignore_file_rel_path, line_no + 1, line),
+                )
+                for line_no, line in enumerate(
+                    map(str.strip, fobj.readlines())
+                )
+                if line and not (line.strip().startswith("#"))
+            ]
+
+        return cls(path_spec_lines, dirname)
 
     def __call__(self, root, dirs, files):
         files = [f for f in files if not self.matches(root, f)]
-        dirs = [d for d in dirs if not self.matches(root, d)]
+        dirs = [d for d in dirs if not self.matches(root, d, True)]
 
         return dirs, files
 
-    def matches(self, dirname, basename):
-        abs_path = os.path.join(dirname, basename)
-        rel_path = relpath(abs_path, self.dirname)
-
-        if os.pardir + os.sep in rel_path:
+    def _get_normalize_path(self, dirname, basename):
+        # NOTE: `relpath` is too slow, so we have to assume that both
+        # `dirname` and `self.dirname` are relative or absolute together.
+        if dirname == self.dirname:
+            path = basename
+        elif dirname.startswith(self.prefix):
+            rel = dirname[len(self.prefix) :]
+            # NOTE: `os.path.join` is ~x5.5 slower
+            path = f"{rel}{os.sep}{basename}"
+        else:
             return False
-        return self.ignore_spec.match_file(rel_path)
+
+        if not System.is_unix():
+            path = normalize_file(path)
+        return path
+
+    def matches(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
+        return self.ignore(path, is_dir)
+
+    def ignore(self, path, is_dir):
+        result = False
+        if is_dir:
+            path_dir = f"{path}/"
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path) or pattern.match(path_dir):
+                    result = ignore
+        else:
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path):
+                    result = ignore
+        return result
+
+    def match_details(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
+        return self._ignore_details(path, is_dir)
+
+    def _ignore_details(self, path, is_dir):
+        result = []
+        for ignore, pattern in zip(self.regex_pattern_list, self.pattern_list):
+            regex = re.compile(ignore[0])
+            # skip system pattern
+            if not pattern.file_info:
+                continue
+            if is_dir:
+                path_dir = f"{path}/"
+                if regex.match(path) or regex.match(path_dir):
+                    result.append(pattern.file_info)
+            else:
+                if regex.match(path):
+                    result.append(pattern.file_info)
+        return result
 
     def __hash__(self):
-        return hash(self.ignore_file_path)
+        return hash(self.dirname + ":" + str(self.pattern_list))
 
     def __eq__(self, other):
         if not isinstance(other, DvcIgnorePatterns):
             return NotImplemented
+        return (self.dirname == other.dirname) & (
+            [pattern.patterns for pattern in self.pattern_list]
+            == [pattern.patterns for pattern in other.pattern_list]
+        )
 
-        return self.ignore_file_path == other.ignore_file_path
+    def __bool__(self):
+        return bool(self.pattern_list)
 
 
-class DvcIgnoreDirs(DvcIgnore):
-    def __init__(self, basenames):
-        self.basenames = set(basenames)
+CheckIgnoreResult = namedtuple(
+    "CheckIgnoreResult", ["file", "match", "patterns"]
+)
 
-    def __call__(self, root, dirs, files):
-        dirs = [d for d in dirs if d not in self.basenames]
 
+def _no_match(path):
+    return CheckIgnoreResult(path, False, ["::"])
+
+
+class DvcIgnoreFilterNoop:
+    def __init__(self, tree, root_dir):
+        pass
+
+    def __call__(self, root, dirs, files, **kwargs):
         return dirs, files
 
-    def __hash__(self):
-        return hash(tuple(self.basenames))
+    def is_ignored_dir(self, _):
+        return False
 
-    def __eq__(self, other):
-        if not isinstance(other, DvcIgnoreDirs):
-            return NotImplemented
+    def is_ignored_file(self, _):
+        return False
 
-        return self.basenames == other.basenames
+    def check_ignore(self, path):
+        return _no_match(path)
+
+    def is_ignored(self, _):
+        return False
 
 
-class DvcIgnoreFilter(object):
-    def __init__(self, tree):
+class DvcIgnoreFilter:
+    @staticmethod
+    def _is_dvc_repo(root, directory):
+        from dvc.repo import Repo
+
+        return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
+
+    def __init__(self, tree, root_dir):
+        from dvc.repo import Repo
+
+        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+
         self.tree = tree
-        self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
-        for root, dirs, files in self.tree.walk(self.tree.tree_root):
-            self._update(root)
-            dirs[:], files[:] = self(root, dirs, files)
+        self.root_dir = root_dir
+        self.ignores_trie_tree = PathStringTrie()
+        self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
+            default_ignore_patterns, root_dir
+        )
+        self._ignored_subrepos = PathStringTrie()
+        self._update(self.root_dir)
 
     def _update(self, dirname):
+        old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
+        matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
+
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if self.tree.exists(ignore_file_path):
-            self.ignores.add(DvcIgnorePatterns(ignore_file_path, self.tree))
+        if not matches and self.tree.exists(
+            ignore_file_path, use_dvcignore=False
+        ):
+            new_pattern = DvcIgnorePatterns.from_files(
+                ignore_file_path, self.tree
+            )
+            if old_pattern:
+                self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
+                    *merge_patterns(
+                        old_pattern.pattern_list,
+                        old_pattern.dirname,
+                        new_pattern.pattern_list,
+                        new_pattern.dirname,
+                    )
+                )
+            else:
+                self.ignores_trie_tree[dirname] = new_pattern
+        elif old_pattern:
+            self.ignores_trie_tree[dirname] = old_pattern
 
-    def __call__(self, root, dirs, files):
-        for ignore in self.ignores:
-            dirs, files = ignore(root, dirs, files)
+        # NOTE: using `walk` + `break` because tree doesn't have `listdir()`
+        for root, dirs, _ in self.tree.walk(dirname, use_dvcignore=False):
+            self._update_sub_repo(root, dirs)
+            break
 
+    def _update_sub_repo(self, root, dirs):
+        for d in dirs:
+            if self._is_dvc_repo(root, d):
+                self._ignored_subrepos[root] = self._ignored_subrepos.get(
+                    root, set()
+                ) | {d}
+                new_pattern = DvcIgnorePatterns([f"/{d}/"], root)
+                old_pattern = self.ignores_trie_tree.longest_prefix(root).value
+                if old_pattern:
+                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
+                        *merge_patterns(
+                            old_pattern.pattern_list,
+                            old_pattern.dirname,
+                            new_pattern.pattern_list,
+                            new_pattern.dirname,
+                        )
+                    )
+                else:
+                    self.ignores_trie_tree[root] = new_pattern
+
+    def __call__(self, root, dirs, files, ignore_subrepos=True):
+        ignore_pattern = self._get_trie_pattern(root)
+        if ignore_pattern:
+            dirs, files = ignore_pattern(root, dirs, files)
+            if not ignore_subrepos:
+                dirs.extend(self._ignored_subrepos.get(root, []))
         return dirs, files
 
+    def _get_trie_pattern(self, dirname):
+        ignore_pattern = self.ignores_trie_tree.get(dirname)
+        if ignore_pattern:
+            return ignore_pattern
 
-class CleanTree(BaseTree):
-    def __init__(self, tree):
-        self.tree = tree
+        prefix = self.ignores_trie_tree.longest_prefix(dirname).key
+        if not prefix:
+            # outside of the repo
+            return None
 
-    @cached_property
-    def dvcignore(self):
-        return DvcIgnoreFilter(self.tree)
+        dirs = list(
+            takewhile(
+                lambda path: path != prefix,
+                (parent.fspath for parent in PathInfo(dirname).parents),
+            )
+        )
+        dirs.reverse()
+        dirs.append(dirname)
 
-    @property
-    def tree_root(self):
-        return self.tree.tree_root
+        for parent in dirs:
+            self._update(parent)
 
-    def open(self, path, mode="r", encoding="utf-8"):
-        return self.tree.open(path, mode, encoding)
+        return self.ignores_trie_tree.get(dirname)
 
-    def exists(self, path):
-        return self.tree.exists(path)
+    def _is_ignored(self, path, is_dir=False):
+        if self._outside_repo(path):
+            return False
+        dirname, basename = os.path.split(os.path.normpath(path))
+        ignore_pattern = self._get_trie_pattern(dirname)
+        if ignore_pattern:
+            return ignore_pattern.matches(dirname, basename, is_dir)
+        else:
+            return False
 
-    def isdir(self, path):
-        return self.tree.isdir(path)
+    def _is_subrepo(self, path):
+        dirname, basename = os.path.split(os.path.normpath(path))
+        return basename in self._ignored_subrepos.get(dirname, set())
 
-    def isfile(self, path):
-        return self.tree.isfile(path)
+    def is_ignored_dir(self, path, ignore_subrepos=True):
+        path = os.path.abspath(path)
+        if not ignore_subrepos:
+            return not self._is_subrepo(path)
+        if path == self.root_dir:
+            return False
 
-    def walk(self, top, topdown=True):
-        for root, dirs, files in self.tree.walk(top, topdown):
-            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+        return self._is_ignored(path, True)
 
-            yield root, dirs, files
+    def is_ignored_file(self, path):
+        path = os.path.abspath(path)
+        return self._is_ignored(path, False)
 
-    def walk_files(self, top):
-        for root, _, files in self.walk(top):
-            for file in files:
-                yield os.path.join(root, file)
+    def _outside_repo(self, path):
+        path = PathInfo(path)
+
+        # paths outside of the repo should be ignored
+        path = relpath(path, self.root_dir)
+        if path.startswith("..") or (
+            os.name == "nt"
+            and not os.path.commonprefix(
+                [os.path.abspath(path), self.root_dir]
+            )
+        ):
+            return True
+        return False
+
+    def check_ignore(self, target):
+        full_target = os.path.abspath(target)
+        if not self._outside_repo(full_target):
+            dirname, basename = os.path.split(os.path.normpath(full_target))
+            pattern = self._get_trie_pattern(dirname)
+            if pattern:
+                matches = pattern.match_details(
+                    dirname, basename, os.path.isdir(full_target)
+                )
+
+                if matches:
+                    return CheckIgnoreResult(target, True, matches)
+        return _no_match(target)
+
+    def is_ignored(self, path):
+        # NOTE: can't use self.check_ignore(path).match for now, see
+        # https://github.com/iterative/dvc/issues/4555
+        return self.is_ignored_dir(path) or self.is_ignored_file(path)
+
+
+def init(path):
+    dvcignore = os.path.join(path, DvcIgnore.DVCIGNORE_FILE)
+    if os.path.exists(dvcignore):
+        return dvcignore
+
+    with open(dvcignore, "w") as fobj:
+        fobj.write(
+            "# Add patterns of files dvc should ignore, which could improve\n"
+            "# the performance. Learn more at\n"
+            "# https://dvc.org/doc/user-guide/dvcignore\n"
+        )
+
+    return dvcignore

dvc/info.py

@@ -0,0 +1,136 @@
+import itertools
+import os
+import pathlib
+import platform
+import uuid
+
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.repo import Repo
+from dvc.scm.base import SCMError
+from dvc.system import System
+from dvc.tree import TREES
+from dvc.utils import error_link
+from dvc.utils.pkg import PKG
+from dvc.version import __version__
+
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+if PKG is None:
+    package = ""
+else:
+    package = f"({PKG})"
+
+
+def get_dvc_info():
+    info = [
+        f"DVC version: {__version__} {package}",
+        "---------------------------------",
+        f"Platform: Python {platform.python_version()} on "
+        f"{platform.platform()}",
+        f"Supports: {_get_supported_remotes()}",
+    ]
+
+    try:
+        repo = Repo()
+
+        # cache_dir might not exist yet (e.g. after `dvc init`), and we
+        # can't auto-create it, as it might cause issues if the user
+        # later decides to enable shared cache mode with
+        # `dvc config cache.shared group`.
+        if os.path.exists(repo.cache.local.cache_dir):
+            info.append(
+                "Cache types: {}".format(_get_linktype_support_info(repo))
+            )
+            if psutil:
+                fs_type = get_fs_type(repo.cache.local.cache_dir)
+                info.append(f"Cache directory: {fs_type}")
+        else:
+            info.append("Cache types: " + error_link("no-dvc-cache"))
+
+    except NotDvcRepoError:
+        pass
+    except SCMError:
+        info.append("Repo: dvc, git (broken)")
+    else:
+        root_directory = repo.root_dir
+        if psutil:
+            fs_root = get_fs_type(os.path.abspath(root_directory))
+            info.append(f"Workspace directory: {fs_root}")
+        info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
+    return "\n".join(info)
+
+
+def _get_linktype_support_info(repo):
+
+    links = {
+        "reflink": (System.reflink, None),
+        "hardlink": (System.hardlink, System.is_hardlink),
+        "symlink": (System.symlink, System.is_symlink),
+    }
+
+    fname = "." + str(uuid.uuid4())
+    src = os.path.join(repo.cache.local.cache_dir, fname)
+    open(src, "w").close()
+    dst = os.path.join(repo.root_dir, fname)
+
+    cache = []
+
+    for name, (link, is_link) in links.items():
+        try:
+            link(src, dst)
+            status = "supported"
+            if is_link and not is_link(dst):
+                status = "broken"
+            os.unlink(dst)
+        except DvcException:
+            status = "not supported"
+
+        if status == "supported":
+            cache.append(name)
+    os.remove(src)
+
+    return ", ".join(cache)
+
+
+def _get_supported_remotes():
+
+    supported_remotes = []
+    for tree_cls in TREES:
+        if not tree_cls.get_missing_deps():
+            supported_remotes.append(tree_cls.scheme)
+
+    if len(supported_remotes) == len(TREES):
+        return "All remotes"
+
+    if len(supported_remotes) == 1:
+        return supported_remotes
+
+    return ", ".join(supported_remotes)
+
+
+def get_fs_type(path):
+
+    partition = {
+        pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device)
+        for part in psutil.disk_partitions(all=True)
+    }
+
+    path = pathlib.Path(path)
+
+    for parent in itertools.chain([path], path.parents):
+        if parent in partition:
+            return partition[parent]
+    return ("unknown", "none")
+
+
+def _get_dvc_repo_info(self):
+    if self.config.get("core", {}).get("no_scm", False):
+        return "dvc (no_scm)"
+
+    if self.root_dir != self.scm.root_dir:
+        return "dvc (subdir), git"
+
+    return "dvc, git"

dvc/istextfile.py

@@ -7,13 +7,17 @@
 TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
 
 
-def istextfile(fname, blocksize=512):
+def istextfile(fname, blocksize=512, tree=None):
     """ Uses heuristics to guess whether the given file is text or binary,
         by reading a single block of bytes from the file.
         If more than 30% of the chars in the block are non-text, or there
         are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
-    with open(fname, "rb") as fobj:
+    if tree:
+        open_func = tree.open
+    else:
+        open_func = open
+    with open_func(fname, "rb") as fobj:
         block = fobj.read(blocksize)
 
     if not block:

dvc/lock.py

@@ -2,22 +2,22 @@
 
 import hashlib
 import os
-import time
 from datetime import timedelta
 
-import zc.lockfile
 import flufl.lock
+import zc.lockfile
+from funcy import retry
 
 from dvc.exceptions import DvcException
-from dvc.utils import format_link
 from dvc.progress import Tqdm
+from dvc.utils import format_link
 
-DEFAULT_TIMEOUT = 5
+DEFAULT_TIMEOUT = 3
 
 FAILED_TO_LOCK_MESSAGE = (
     "cannot perform the command because another DVC process seems to be "
     "running on this project. If that is not the case, manually remove "
-    "`.dvc/lock` and try again."
+    "`.dvc/tmp/lock` and try again."
 )
 
 
@@ -25,7 +25,7 @@ class LockError(DvcException):
     """Thrown when unable to acquire the lock for DVC repo."""
 
 
-class Lock(object):
+class Lock:
     """Class for DVC repo lock.
 
     Uses zc.lockfile as backend.
@@ -47,7 +47,7 @@ class Lock(object):
                 disable=not self._friendly,
                 desc=(
                     "If DVC froze, see `hardlink_lock` in {}".format(
-                        format_link("man.dvc.org/config#core")
+                        format_link("https://man.dvc.org/config#core")
                     )
                 ),
             ):
@@ -56,13 +56,10 @@ class Lock(object):
             raise LockError(FAILED_TO_LOCK_MESSAGE)
 
     def lock(self):
-        try:
-            self._do_lock()
-            return
-        except LockError:
-            time.sleep(DEFAULT_TIMEOUT)
-
-        self._do_lock()
+        retries = 6
+        delay = DEFAULT_TIMEOUT / retries
+        lock_retry = retry(retries, LockError, timeout=delay)(self._do_lock)
+        lock_retry()
 
     def unlock(self):
         self._lock.close()
@@ -88,7 +85,9 @@ class HardlinkLock(flufl.lock.Lock):
         tmp_dir (str): a directory to store claim files.
     """
 
-    def __init__(self, lockfile, tmp_dir=None, **kwargs):
+    def __init__(
+        self, lockfile, tmp_dir=None, **kwargs
+    ):  # pylint: disable=super-init-not-called
         import socket
 
         self._tmp_dir = tmp_dir
@@ -117,7 +116,7 @@ class HardlinkLock(flufl.lock.Lock):
     def lockfile(self):
         return self._lockfile
 
-    def lock(self):
+    def lock(self):  # pylint: disable=arguments-differ
         try:
             super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
         except flufl.lock.TimeOutError:

dvc/logger.py

@@ -7,7 +7,6 @@ import colorama
 
 from dvc.progress import Tqdm
 
-
 FOOTER = (
     "\n{yellow}Having any troubles?{nc}"
     " Hit us up at {blue}https://dvc.org/support{nc},"
@@ -19,20 +18,46 @@ FOOTER = (
 )
 
 
+def addLoggingLevel(levelName, levelNum, methodName=None):
+    """
+    Adds a new logging level to the `logging` module and the
+    currently configured logging class.
+
+    Based on https://stackoverflow.com/questions/2183233
+    """
+    if methodName is None:
+        methodName = levelName.lower()
+
+    assert not hasattr(logging, levelName)
+    assert not hasattr(logging, methodName)
+    assert not hasattr(logging.getLoggerClass(), methodName)
+
+    def logForLevel(self, message, *args, **kwargs):
+        if self.isEnabledFor(levelNum):
+            # pylint: disable=protected-access
+            self._log(levelNum, message, args, **kwargs)
+
+    def logToRoot(message, *args, **kwargs):
+        logging.log(levelNum, message, *args, **kwargs)
+
+    logging.addLevelName(levelNum, levelName)
+    setattr(logging, levelName, levelNum)
+    setattr(logging.getLoggerClass(), methodName, logForLevel)
+    setattr(logging, methodName, logToRoot)
+
+
 class LoggingException(Exception):
     def __init__(self, record):
         msg = "failed to log {}".format(str(record))
         super().__init__(msg)
 
 
-class ExcludeErrorsFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.WARNING
+def excludeFilter(level):
+    class ExcludeLevelFilter(logging.Filter):
+        def filter(self, record):
+            return record.levelno < level
 
-
-class ExcludeInfoFilter(logging.Filter):
-    def filter(self, record):
-        return record.levelno < logging.INFO
+    return ExcludeLevelFilter
 
 
 class ColorFormatter(logging.Formatter):
@@ -48,6 +73,7 @@ class ColorFormatter(logging.Formatter):
     """
 
     color_code = {
+        "TRACE": colorama.Fore.GREEN,
         "DEBUG": colorama.Fore.BLUE,
         "WARNING": colorama.Fore.YELLOW,
         "ERROR": colorama.Fore.RED,
@@ -55,7 +81,8 @@ class ColorFormatter(logging.Formatter):
     }
 
     def format(self, record):
-        msg = record.msg.format(*record.args) if record.args else record.msg
+        record.message = record.getMessage()
+        msg = self.formatMessage(record)
 
         if record.levelname == "INFO":
             return msg
@@ -111,12 +138,16 @@ class LoggerHandler(logging.StreamHandler):
             self.flush()
         except RecursionError:
             raise
-        except Exception:
+        except Exception:  # noqa, pylint: disable=broad-except
             self.handleError(record)
 
 
 def _is_verbose():
-    return logging.getLogger("dvc").getEffectiveLevel() == logging.DEBUG
+    return (
+        logging.NOTSET
+        < logging.getLogger("dvc").getEffectiveLevel()
+        <= logging.DEBUG
+    )
 
 
 def _iter_causes(exc):
@@ -141,15 +172,25 @@ def _stack_trace(exc_info):
     )
 
 
+def disable_other_loggers():
+    logging.captureWarnings(True)
+    root = logging.root
+    for (logger_name, logger) in root.manager.loggerDict.items():
+        if logger_name != "dvc" and not logger_name.startswith("dvc."):
+            logger.disabled = True
+
+
 def setup(level=logging.INFO):
     colorama.init()
 
+    addLoggingLevel("TRACE", logging.DEBUG - 5)
     logging.config.dictConfig(
         {
             "version": 1,
             "filters": {
-                "exclude_errors": {"()": ExcludeErrorsFilter},
-                "exclude_info": {"()": ExcludeInfoFilter},
+                "exclude_errors": {"()": excludeFilter(logging.WARNING)},
+                "exclude_info": {"()": excludeFilter(logging.INFO)},
+                "exclude_debug": {"()": excludeFilter(logging.DEBUG)},
             },
             "formatters": {"color": {"()": ColorFormatter}},
             "handlers": {
@@ -167,6 +208,13 @@ def setup(level=logging.INFO):
                     "stream": "ext://sys.stdout",
                     "filters": ["exclude_info"],
                 },
+                "console_trace": {
+                    "class": "dvc.logger.LoggerHandler",
+                    "level": "TRACE",
+                    "formatter": "color",
+                    "stream": "ext://sys.stdout",
+                    "filters": ["exclude_debug"],
+                },
                 "console_errors": {
                     "class": "dvc.logger.LoggerHandler",
                     "level": "WARNING",
@@ -180,25 +228,11 @@ def setup(level=logging.INFO):
                     "handlers": [
                         "console_info",
                         "console_debug",
-                        "console_errors",
-                    ],
-                },
-                "paramiko": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
-                        "console_errors",
-                    ],
-                },
-                "flufl.lock": {
-                    "level": logging.CRITICAL,
-                    "handlers": [
-                        "console_info",
-                        "console_debug",
+                        "console_trace",
                         "console_errors",
                     ],
                 },
             },
+            "disable_existing_loggers": False,
         }
     )

dvc/main.py

@@ -2,16 +2,16 @@
 
 import errno
 import logging
+from contextlib import contextmanager
 
 from dvc import analytics
 from dvc.cli import parse_args
 from dvc.config import ConfigError
-from dvc.exceptions import DvcParserError
-from dvc.exceptions import NotDvcRepoError
+from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
-from dvc.logger import FOOTER
-from dvc.utils import format_link
-from dvc.remote.pool import close_pools
+from dvc.logger import FOOTER, disable_other_loggers
+from dvc.tree.pool import close_pools
+from dvc.utils import error_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
@@ -19,11 +19,30 @@ from dvc.remote.pool import close_pools
 
 "".encode("idna")
 
-
 logger = logging.getLogger("dvc")
 
 
-def main(argv=None):
+@contextmanager
+def profile(enable, dump):
+    if not enable:
+        yield
+        return
+
+    import cProfile
+
+    prof = cProfile.Profile()
+    prof.enable()
+
+    yield
+
+    prof.disable()
+    if not dump:
+        prof.print_stats(sort="cumtime")
+        return
+    prof.dump_stats(dump)
+
+
+def main(argv=None):  # noqa: C901
     """Run dvc CLI command.
 
     Args:
@@ -33,20 +52,28 @@ def main(argv=None):
         int: command's return code.
     """
     args = None
-    cmd = None
+    disable_other_loggers()
 
     outerLogLevel = logger.level
     try:
         args = parse_args(argv)
 
+        level = None
         if args.quiet:
-            logger.setLevel(logging.CRITICAL)
+            level = logging.CRITICAL
+        elif args.verbose == 1:
+            level = logging.DEBUG
+        elif args.verbose > 1:
+            level = logging.TRACE
 
-        elif args.verbose:
-            logger.setLevel(logging.DEBUG)
+        if level is not None:
+            logger.setLevel(level)
 
-        cmd = args.func(args)
-        ret = cmd.run()
+        logger.trace(args)
+
+        with profile(enable=args.cprofile, dump=args.cprofile_dump):
+            cmd = args.func(args)
+            ret = cmd.run()
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -58,23 +85,36 @@ def main(argv=None):
         ret = 253
     except DvcParserError:
         ret = 254
-    except Exception as exc:  # pylint: disable=broad-except
+    except DvcException:
+        ret = 255
+        logger.exception("")
+    except Exception as exc:  # noqa, pylint: disable=broad-except
+        # pylint: disable=no-member
         if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
             logger.exception(
                 "too many open files, please visit "
                 "{} to see how to handle this "
-                "problem".format(
-                    format_link("https://error.dvc.org/many-files")
-                ),
+                "problem".format(error_link("many-files")),
                 extra={"tb_only": True},
             )
         else:
+            from dvc.info import get_dvc_info
+
             logger.exception("unexpected error")
+            logger.info(FOOTER)
+            dvc_info = get_dvc_info()
+            logger.info(dvc_info)
         ret = 255
+
+    try:
+        if analytics.is_enabled():
+            analytics.collect_and_send_report(args, ret)
+
+        return ret
     finally:
         logger.setLevel(outerLogLevel)
 
-        # Closing pools by-hand to prevent wierd messages when closing SSH
+        # Closing pools by-hand to prevent weird messages when closing SSH
         # connections. See https://github.com/iterative/dvc/issues/3248 for
         # more info.
         close_pools()
@@ -82,11 +122,3 @@ def main(argv=None):
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()
-
-    if ret != 0:
-        logger.info(FOOTER)
-
-    if analytics.is_enabled():
-        analytics.collect_and_send_report(args, ret)
-
-    return ret

dvc/output/__init__.py

@@ -1,32 +1,36 @@
+from collections import defaultdict
 from urllib.parse import urlparse
-from voluptuous import Any, Required, Lower, Length, Coerce, And, SetTo
 
-from dvc.output.base import OutputBase
-from dvc.output.gs import OutputGS
-from dvc.output.hdfs import OutputHDFS
-from dvc.output.local import OutputLOCAL
-from dvc.output.s3 import OutputS3
-from dvc.output.ssh import OutputSSH
-from dvc.remote import Remote
-from dvc.remote.hdfs import RemoteHDFS
-from dvc.remote.local import RemoteLOCAL
-from dvc.remote.s3 import RemoteS3
+from funcy import collecting, project
+from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
+
+from dvc.output.base import BaseOutput
+from dvc.output.gs import GSOutput
+from dvc.output.hdfs import HDFSOutput
+from dvc.output.local import LocalOutput
+from dvc.output.s3 import S3Output
+from dvc.output.ssh import SSHOutput
 from dvc.scheme import Schemes
 
+from ..tree import get_cloud_tree
+from ..tree.hdfs import HDFSTree
+from ..tree.local import LocalTree
+from ..tree.s3 import S3Tree
+
 OUTS = [
-    OutputHDFS,
-    OutputS3,
-    OutputGS,
-    OutputSSH,
-    # NOTE: OutputLOCAL is the default choice
+    HDFSOutput,
+    S3Output,
+    GSOutput,
+    SSHOutput,
+    # NOTE: LocalOutput is the default choice
 ]
 
 OUTS_MAP = {
-    Schemes.HDFS: OutputHDFS,
-    Schemes.S3: OutputS3,
-    Schemes.GS: OutputGS,
-    Schemes.SSH: OutputSSH,
-    Schemes.LOCAL: OutputLOCAL,
+    Schemes.HDFS: HDFSOutput,
+    Schemes.S3: S3Output,
+    Schemes.GS: GSOutput,
+    Schemes.SSH: SSHOutput,
+    Schemes.LOCAL: LocalOutput,
 }
 
 CHECKSUM_SCHEMA = Any(
@@ -44,35 +48,35 @@ CHECKSUM_SCHEMA = Any(
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUMS_SCHEMA = {
-    RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }
 
-TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}
-
 SCHEMA = CHECKSUMS_SCHEMA.copy()
-SCHEMA[Required(OutputBase.PARAM_PATH)] = str
-SCHEMA[OutputBase.PARAM_CACHE] = bool
-SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
-SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA
-SCHEMA[OutputBase.PARAM_PERSIST] = bool
+SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
+SCHEMA[BaseOutput.PARAM_CACHE] = bool
+SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
+SCHEMA[BaseOutput.PARAM_PLOT] = bool
+SCHEMA[BaseOutput.PARAM_PERSIST] = bool
 
 
-def _get(stage, p, info, cache, metric, persist=False, tags=None):
+def _get(
+    stage, p, info=None, cache=True, metric=False, plot=False, persist=False
+):
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        remote = Remote(stage.repo, name=parsed.netloc)
-        return OUTS_MAP[remote.scheme](
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return OUTS_MAP[tree.scheme](
             stage,
             p,
             info,
             cache=cache,
-            remote=remote,
+            tree=tree,
             metric=metric,
+            plot=plot,
             persist=persist,
-            tags=tags,
         )
 
     for o in OUTS:
@@ -82,31 +86,31 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
                 p,
                 info,
                 cache=cache,
-                remote=None,
+                tree=None,
                 metric=metric,
+                plot=plot,
                 persist=persist,
-                tags=tags,
             )
-    return OutputLOCAL(
+    return LocalOutput(
         stage,
         p,
         info,
         cache=cache,
-        remote=None,
+        tree=None,
         metric=metric,
+        plot=plot,
         persist=persist,
-        tags=tags,
     )
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(OutputBase.PARAM_PATH)
-        cache = d.pop(OutputBase.PARAM_CACHE, True)
-        metric = d.pop(OutputBase.PARAM_METRIC, False)
-        persist = d.pop(OutputBase.PARAM_PERSIST, False)
-        tags = d.pop(OutputBase.PARAM_TAGS, None)
+        p = d.pop(BaseOutput.PARAM_PATH)
+        cache = d.pop(BaseOutput.PARAM_CACHE, True)
+        metric = d.pop(BaseOutput.PARAM_METRIC, False)
+        plot = d.pop(BaseOutput.PARAM_PLOT, False)
+        persist = d.pop(BaseOutput.PARAM_PERSIST, False)
         ret.append(
             _get(
                 stage,
@@ -114,24 +118,67 @@ def loadd_from(stage, d_list):
                 info=d,
                 cache=cache,
                 metric=metric,
+                plot=plot,
                 persist=persist,
-                tags=tags,
             )
         )
     return ret
 
 
-def loads_from(stage, s_list, use_cache=True, metric=False, persist=False):
-    ret = []
-    for s in s_list:
-        ret.append(
-            _get(
-                stage,
-                s,
-                info={},
-                cache=use_cache,
-                metric=metric,
-                persist=persist,
-            )
+def loads_from(
+    stage, s_list, use_cache=True, metric=False, plot=False, persist=False
+):
+    return [
+        _get(
+            stage,
+            s,
+            info={},
+            cache=use_cache,
+            metric=metric,
+            plot=plot,
+            persist=persist,
         )
-    return ret
+        for s in s_list
+    ]
+
+
+def _split_dict(d, keys):
+    return project(d, keys), project(d, d.keys() - keys)
+
+
+def _merge_data(s_list):
+    d = defaultdict(dict)
+    for key in s_list:
+        if isinstance(key, str):
+            d[key].update({})
+            continue
+        if not isinstance(key, dict):
+            raise ValueError(f"'{type(key).__name__}' not supported.")
+
+        for k, flags in key.items():
+            if not isinstance(flags, dict):
+                raise ValueError(
+                    f"Expected dict for '{k}', got: '{type(flags).__name__}'"
+                )
+            d[k].update(flags)
+    return d
+
+
+@collecting
+def load_from_pipeline(stage, s_list, typ="outs"):
+    if typ not in (stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS):
+        raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
+
+    metric = typ == stage.PARAM_METRICS
+    plot = typ == stage.PARAM_PLOTS
+
+    d = _merge_data(s_list)
+
+    for path, flags in d.items():
+        plt_d = {}
+        if plot:
+            from dvc.schema import PLOT_PROPS
+
+            plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
+        extra = project(flags, ["cache", "persist"])
+        yield _get(stage, path, {}, plot=plt_d or plot, metric=metric, **extra)

dvc/output/base.py

@@ -1,52 +1,76 @@
 import logging
-from urllib.parse import urlparse
+import os
 from copy import copy
+from urllib.parse import urlparse
 
 from voluptuous import Any
 
 import dvc.prompt as prompt
 from dvc.cache import NamedCache
-from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
-from dvc.exceptions import DvcException
-from dvc.remote.base import RemoteBASE
+from dvc.exceptions import (
+    CollectCacheError,
+    DvcException,
+    MergeError,
+    RemoteCacheRequiredError,
+)
+from dvc.hash_info import HashInfo
 
+from ..tree.base import BaseTree
 
 logger = logging.getLogger(__name__)
 
 
 class OutputDoesNotExistError(DvcException):
     def __init__(self, path):
-        msg = "output '{}' does not exist".format(path)
+        msg = f"output '{path}' does not exist"
         super().__init__(msg)
 
 
 class OutputIsNotFileOrDirError(DvcException):
     def __init__(self, path):
-        msg = "output '{}' is not a file or directory".format(path)
+        msg = f"output '{path}' is not a file or directory"
         super().__init__(msg)
 
 
 class OutputAlreadyTrackedError(DvcException):
     def __init__(self, path):
-        msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
+        msg = f""" output '{path}' is already tracked by SCM (e.g. Git).
+    You can remove it from Git, then add to DVC.
+        To stop tracking from Git:
+            git rm -r --cached '{path}'
+            git commit -m "stop tracking {path}" """
         super().__init__(msg)
 
 
 class OutputIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__("Stage file '{}' cannot be an output.".format(path))
+        super().__init__(f"Stage file '{path}' cannot be an output.")
 
 
-class OutputBase(object):
+class OutputIsIgnoredError(DvcException):
+    def __init__(self, match):
+        lines = "\n".join(match.patterns)
+        super().__init__(f"Path '{match.file}' is ignored by\n{lines}")
+
+
+class BaseOutput:
     IS_DEPENDENCY = False
 
-    REMOTE = RemoteBASE
+    TREE_CLS = BaseTree
 
     PARAM_PATH = "path"
     PARAM_CACHE = "cache"
     PARAM_METRIC = "metric"
     PARAM_METRIC_TYPE = "type"
     PARAM_METRIC_XPATH = "xpath"
+    PARAM_PLOT = "plot"
+    PARAM_PLOT_TEMPLATE = "template"
+    PARAM_PLOT_X = "x"
+    PARAM_PLOT_Y = "y"
+    PARAM_PLOT_X_LABEL = "x_label"
+    PARAM_PLOT_Y_LABEL = "y_label"
+    PARAM_PLOT_TITLE = "title"
+    PARAM_PLOT_HEADER = "header"
     PARAM_PERSIST = "persist"
 
     METRIC_SCHEMA = Any(
@@ -58,11 +82,10 @@ class OutputBase(object):
         },
     )
 
-    PARAM_TAGS = "tags"
-
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
     IsStageFileError = OutputIsStageFileError
+    IsIgnoredError = OutputIsIgnoredError
 
     sep = "/"
 
@@ -71,13 +94,13 @@ class OutputBase(object):
         stage,
         path,
         info=None,
-        remote=None,
+        tree=None,
         cache=True,
         metric=False,
+        plot=False,
         persist=False,
-        tags=None,
     ):
-        self._validate_output_path(path)
+        self._validate_output_path(path, stage)
         # This output (and dependency) objects have too many paths/urls
         # here is a list and comments:
         #
@@ -91,22 +114,25 @@ class OutputBase(object):
         self.stage = stage
         self.repo = stage.repo if stage else None
         self.def_path = path
-        self.info = info
-        self.remote = remote or self.REMOTE(self.repo, {})
+        self.hash_info = HashInfo.from_dict(info)
+        if tree:
+            self.tree = tree
+        else:
+            self.tree = self.TREE_CLS(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
+        self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
-        self.tags = None if self.IS_DEPENDENCY else (tags or {})
 
-        self.path_info = self._parse_path(remote, path)
+        self.path_info = self._parse_path(tree, path)
         if self.use_cache and self.cache is None:
             raise RemoteCacheRequiredError(self.path_info)
 
-    def _parse_path(self, remote, path):
-        if remote:
+    def _parse_path(self, tree, path):
+        if tree:
             parsed = urlparse(path)
-            return remote.path_info / parsed.path.lstrip("/")
-        return self.REMOTE.path_cls(path)
+            return tree.path_info / parsed.path.lstrip("/")
+        return self.TREE_CLS.PATH_CLS(path)
 
     def __repr__(self):
         return "{class_name}: '{def_path}'".format(
@@ -118,7 +144,7 @@ class OutputBase(object):
 
     @property
     def scheme(self):
-        return self.REMOTE.scheme
+        return self.TREE_CLS.scheme
 
     @property
     def is_in_repo(self):
@@ -137,52 +163,58 @@ class OutputBase(object):
 
     @property
     def dir_cache(self):
-        return self.cache.get_dir_cache(self.checksum)
+        return self.cache.get_dir_cache(self.hash_info)
 
     @classmethod
     def supported(cls, url):
-        return cls.REMOTE.supported(url)
+        return cls.TREE_CLS.supported(url)
 
     @property
     def cache_path(self):
-        return self.cache.checksum_to_path_info(self.checksum).url
+        return self.cache.tree.hash_to_path_info(self.hash_info.value).url
 
-    @property
-    def checksum(self):
-        return self.info.get(self.remote.PARAM_CHECKSUM)
+    def get_hash(self):
+        if not self.use_cache:
+            return self.tree.get_hash(self.path_info)
+        return self.cache.get_hash(self.tree, self.path_info)
 
     @property
     def is_dir_checksum(self):
-        return self.remote.is_dir_checksum(self.checksum)
+        return self.hash_info.isdir
 
     @property
     def exists(self):
-        return self.remote.exists(self.path_info)
+        return self.tree.exists(self.path_info)
 
     def changed_checksum(self):
-        return self.checksum != self.remote.get_checksum(self.path_info)
+        return self.hash_info != self.get_hash()
 
     def changed_cache(self, filter_info=None):
-        if not self.use_cache or not self.checksum:
+        if not self.use_cache or not self.hash_info:
             return True
 
-        return self.cache.changed_cache(self.checksum, filter_info=filter_info)
-
-    def status(self):
-        if self.checksum and self.use_cache and self.changed_cache():
-            return {str(self): "not in cache"}
+        return self.cache.changed_cache(
+            self.hash_info, filter_info=filter_info
+        )
 
+    def workspace_status(self):
         if not self.exists:
             return {str(self): "deleted"}
 
         if self.changed_checksum():
             return {str(self): "modified"}
 
-        if not self.checksum:
+        if not self.hash_info:
             return {str(self): "new"}
 
         return {}
 
+    def status(self):
+        if self.hash_info and self.use_cache and self.changed_cache():
+            return {str(self): "not in cache"}
+
+        return self.workspace_status()
+
     def changed(self):
         status = self.status()
         logger.debug(str(status))
@@ -190,13 +222,32 @@ class OutputBase(object):
 
     @property
     def is_empty(self):
-        return self.remote.is_empty(self.path_info)
+        return self.tree.is_empty(self.path_info)
 
     def isdir(self):
-        return self.remote.isdir(self.path_info)
+        return self.tree.isdir(self.path_info)
 
     def isfile(self):
-        return self.remote.isfile(self.path_info)
+        return self.tree.isfile(self.path_info)
+
+    # pylint: disable=no-member
+
+    def ignore(self):
+        if not self.use_scm_ignore:
+            return
+
+        if self.repo.scm.is_tracked(self.fspath):
+            raise OutputAlreadyTrackedError(self)
+
+        self.repo.scm.ignore(self.fspath)
+
+    def ignore_remove(self):
+        if not self.use_scm_ignore:
+            return
+
+        self.repo.scm.ignore_remove(self.fspath)
+
+    # pylint: enable=no-member
 
     def save(self):
         if not self.exists:
@@ -206,48 +257,43 @@ class OutputBase(object):
             raise self.IsNotFileOrDirError(self)
 
         if self.is_empty:
-            logger.warning("'{}' is empty.".format(self))
+            logger.warning(f"'{self}' is empty.")
 
-        if self.use_scm_ignore:
-            if self.repo.scm.is_tracked(self.fspath):
-                raise OutputAlreadyTrackedError(self)
+        self.ignore()
 
-            self.repo.scm.ignore(self.fspath)
+        if self.metric or self.plot:
+            self.verify_metric()
 
         if not self.use_cache:
-            self.info = self.remote.save_info(self.path_info)
-            if self.metric:
-                self.verify_metric()
+            self.hash_info = self.get_hash()
             if not self.IS_DEPENDENCY:
-                logger.info(
-                    "Output '{}' doesn't use cache. Skipping saving.".format(
-                        self
-                    )
+                logger.debug(
+                    "Output '%s' doesn't use cache. Skipping saving.", self
                 )
             return
 
         assert not self.IS_DEPENDENCY
 
         if not self.changed():
-            logger.info(
-                "Output '{}' didn't change. Skipping saving.".format(self)
-            )
+            logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
-        self.info = self.remote.save_info(self.path_info)
+        self.hash_info = self.get_hash()
 
     def commit(self):
+        assert self.hash_info
         if self.use_cache:
-            self.cache.save(self.path_info, self.info)
+            self.cache.save(self.path_info, self.cache.tree, self.hash_info)
 
     def dumpd(self):
-        ret = copy(self.info)
+        ret = copy(self.hash_info.to_dict())
         ret[self.PARAM_PATH] = self.def_path
 
         if self.IS_DEPENDENCY:
             return ret
 
-        ret[self.PARAM_CACHE] = self.use_cache
+        if not self.use_cache:
+            ret[self.PARAM_CACHE] = self.use_cache
 
         if isinstance(self.metric, dict):
             if (
@@ -256,27 +302,27 @@ class OutputBase(object):
             ):
                 del self.metric[self.PARAM_METRIC_XPATH]
 
-        ret[self.PARAM_METRIC] = self.metric
-        ret[self.PARAM_PERSIST] = self.persist
+        if self.metric:
+            ret[self.PARAM_METRIC] = self.metric
 
-        if self.tags:
-            ret[self.PARAM_TAGS] = self.tags
+        if self.plot:
+            ret[self.PARAM_PLOT] = self.plot
+
+        if self.persist:
+            ret[self.PARAM_PERSIST] = self.persist
 
         return ret
 
     def verify_metric(self):
-        raise DvcException(
-            "verify metric is not supported for {}".format(self.scheme)
-        )
+        raise DvcException(f"verify metric is not supported for {self.scheme}")
 
     def download(self, to):
-        self.remote.download(self.path_info, to.path_info)
+        self.tree.download(self.path_info, to.path_info)
 
     def checkout(
         self,
         force=False,
         progress_callback=None,
-        tag=None,
         relink=False,
         filter_info=None,
     ):
@@ -287,14 +333,9 @@ class OutputBase(object):
                 )
             return None
 
-        if tag:
-            info = self.tags[tag]
-        else:
-            info = self.info
-
         return self.cache.checkout(
             self.path_info,
-            info,
+            self.hash_info,
             force=force,
             progress_callback=progress_callback,
             relink=relink,
@@ -302,18 +343,19 @@ class OutputBase(object):
         )
 
     def remove(self, ignore_remove=False):
-        self.remote.remove(self.path_info)
+        self.tree.remove(self.path_info)
         if self.scheme != "local":
             return
 
-        if ignore_remove and self.use_scm_ignore:
-            self.repo.scm.ignore_remove(self.fspath)
+        if ignore_remove:
+            self.ignore_remove()
 
     def move(self, out):
+        # pylint: disable=no-member
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore_remove(self.fspath)
 
-        self.remote.move(self.path_info, out.path_info)
+        self.tree.move(self.path_info, out.path_info)
         self.def_path = out.def_path
         self.path_info = out.path_info
         self.save()
@@ -327,14 +369,25 @@ class OutputBase(object):
             return 0
 
         return self.cache.get_files_number(
-            self.path_info, self.checksum, filter_info
+            self.path_info, self.hash_info, filter_info
         )
 
     def unprotect(self):
         if self.exists:
-            self.remote.unprotect(self.path_info)
+            self.tree.unprotect(self.path_info)
 
-    def _collect_used_dir_cache(
+    def get_dir_cache(self, **kwargs):
+        if not self.is_dir_checksum:
+            raise DvcException("cannot get dir cache for file checksum")
+        if self.cache.changed_cache_file(self.hash_info):
+            self.repo.cloud.pull(
+                NamedCache.make("local", self.hash_info.value, str(self)),
+                show_checksums=False,
+                **kwargs,
+            )
+        return self.dir_cache
+
+    def collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
     ):
         """Get a list of `info`s related to the given directory.
@@ -358,18 +411,12 @@ class OutputBase(object):
 
         cache = NamedCache()
 
-        if self.cache.changed_cache_file(self.checksum):
-            try:
-                self.repo.cloud.pull(
-                    NamedCache.make("local", self.checksum, str(self)),
-                    jobs=jobs,
-                    remote=remote,
-                    show_checksums=False,
-                )
-            except DvcException:
-                logger.debug("failed to pull cache for '{}'".format(self))
+        try:
+            self.get_dir_cache(jobs=jobs, remote=remote)
+        except DvcException:
+            logger.debug(f"failed to pull cache for '{self}'")
 
-        if self.cache.changed_cache_file(self.checksum):
+        if self.cache.changed_cache_file(self.hash_info):
             msg = (
                 "Missing cache for directory '{}'. "
                 "Cache for files inside will be lost. "
@@ -383,11 +430,21 @@ class OutputBase(object):
             else:
                 return cache
 
+        path = str(self.path_info)
+        filter_path = str(filter_info) if filter_info else None
+        is_win = os.name == "nt"
         for entry in self.dir_cache:
-            checksum = entry[self.remote.PARAM_CHECKSUM]
-            info = self.path_info / entry[self.remote.PARAM_RELPATH]
-            if not filter_info or info.isin_or_eq(filter_info):
-                cache.add(self.scheme, checksum, str(info))
+            checksum = entry[self.tree.PARAM_CHECKSUM]
+            entry_relpath = entry[self.tree.PARAM_RELPATH]
+            if is_win:
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            entry_path = os.path.join(path, entry_relpath)
+            if (
+                not filter_path
+                or entry_path == filter_path
+                or entry_path.startswith(filter_path + os.sep)
+            ):
+                cache.add(self.scheme, checksum, entry_path)
 
         return cache
 
@@ -409,26 +466,80 @@ class OutputBase(object):
             cache.external[dep.repo_pair].add(dep.def_path)
             return cache
 
-        if not self.info:
-            logger.warning(
-                "Output '{}'({}) is missing version info. Cache for it will "
-                "not be collected. Use `dvc repro` to get your pipeline up to "
-                "date.".format(self, self.stage)
+        if not self.hash_info:
+            msg = (
+                "Output '{}'({}) is missing version info. "
+                "Cache for it will not be collected. "
+                "Use `dvc repro` to get your pipeline up to date.".format(
+                    self, self.stage
+                )
             )
+            if self.exists:
+                msg += (
+                    "\n"
+                    "You can also use `dvc commit {stage.addressing}` "
+                    "to associate existing '{out}' with {stage}.".format(
+                        out=self, stage=self.stage
+                    )
+                )
+            logger.warning(msg)
             return NamedCache()
 
-        ret = NamedCache.make(self.scheme, self.checksum, str(self))
+        ret = NamedCache.make(self.scheme, self.hash_info.value, str(self))
 
         if not self.is_dir_checksum:
             return ret
 
-        ret.update(self._collect_used_dir_cache(**kwargs))
+        ret.add_child_cache(
+            self.hash_info.value, self.collect_used_dir_cache(**kwargs),
+        )
 
         return ret
 
     @classmethod
-    def _validate_output_path(cls, path):
-        from dvc.stage import Stage
+    def _validate_output_path(cls, path, stage=None):
+        from dvc.dvcfile import is_valid_filename
 
-        if Stage.is_valid_filename(path):
+        if is_valid_filename(path):
             raise cls.IsStageFileError(path)
+
+        if stage:
+            check = stage.repo.tree.dvcignore.check_ignore(path)
+            if check.match:
+                raise cls.IsIgnoredError(check)
+
+    def _check_can_merge(self, out):
+        if self.scheme != out.scheme:
+            raise MergeError("unable to auto-merge outputs of different types")
+
+        my = self.dumpd()
+        other = out.dumpd()
+
+        my.pop(self.tree.PARAM_CHECKSUM)
+        other.pop(self.tree.PARAM_CHECKSUM)
+
+        if my != other:
+            raise MergeError(
+                "unable to auto-merge outputs with different options"
+            )
+
+        if not out.is_dir_checksum:
+            raise MergeError(
+                "unable to auto-merge outputs that are not directories"
+            )
+
+    def merge(self, ancestor, other):
+        assert other
+
+        if ancestor:
+            self._check_can_merge(ancestor)
+            ancestor_info = ancestor.hash_info
+        else:
+            ancestor_info = None
+
+        self._check_can_merge(self)
+        self._check_can_merge(other)
+
+        self.hash_info = self.cache.merge(
+            ancestor_info, self.hash_info, other.hash_info
+        )

dvc/output/gs.py

@@ -1,6 +1,7 @@
-from dvc.output.s3 import OutputS3
-from dvc.remote.gs import RemoteGS
+from dvc.output.s3 import S3Output
+
+from ..tree.gs import GSTree
 
 
-class OutputGS(OutputS3):
-    REMOTE = RemoteGS
+class GSOutput(S3Output):
+    TREE_CLS = GSTree

dvc/output/hdfs.py

@@ -1,6 +1,7 @@
-from dvc.output.base import OutputBase
-from dvc.remote.hdfs import RemoteHDFS
+from dvc.output.base import BaseOutput
+
+from ..tree.hdfs import HDFSTree
 
 
-class OutputHDFS(OutputBase):
-    REMOTE = RemoteHDFS
+class HDFSOutput(BaseOutput):
+    TREE_CLS = HDFSTree