dvc: bump to 1.11.3

.github/ISSUE_TEMPLATE/bug_report.md

@@ -3,62 +3,16 @@ name: "\U0001F41B Bug Report"
 about: Create a bug report to help us improve DVC
 ---
 
-# Bug Report
+## Bug Report
 
-<!--
-## Issue name
+### Please provide information about your setup
 
-Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug. 
-
-Example: `repro: doesn't detect input changes`
--->
-
-## Description
-
-<!--
-A clear and concise description of what the bug is.
--->
-
-### Reproduce
-
-<!--
-Step list of how to reproduce the bug
--->
-
-<!--
-Example:
-
-1. dvc init
-2. Copy dataset.zip to the directory
-3. dvc add dataset.zip
-4. dvc run -d dataset.zip -o model ./train.sh
-5. modify dataset.zip
-6. dvc repro
--->
-
-### Expected
-
-<!--
-A clear and concise description of what you expect to happen.
--->
-
-### Environment information
-
-<!--
-This is required to ensure that we can reproduce the bug.
--->
-
-**Output of `dvc doctor`:**
+**Output of `dvc version`:**
 
 ```console
-$ dvc doctor
+$ dvc version
 ```
 
 **Additional Information (if any):**
 
-<!--
-Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
-
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
-If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
--->

.github/mergify.yml

@@ -1,9 +0,0 @@
-pull_request_rules:
-  - name: backport patches to 1.11
-    conditions:
-      - base=master
-      - label=backport-1.11
-    actions:
-      backport:
-        branches:
-          - "1.11"

.github/workflows/packages.yaml

@@ -1,35 +1,23 @@
 name: Build packages
 on:
-  push:
-    branches:
-    - master
+  push: {}
   release:
-    types: [released, prereleased]
+    types: [released]
 jobs:
-  deb:
+  linux:
     runs-on: ubuntu-16.04
     steps:
     - uses: actions/[email protected]
-    - name: Set up Python 3.8
+    - name: Set up Python 3.7
       uses: actions/[email protected]
       with:
-        python-version: 3.8
+        python-version: 3.7
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Install deps
-      run: |
-        pip install .[all]
-        pip install -r scripts/build-requirements.txt
-        gem install --no-document fpm
-    - name: Build deb
-      run: python scripts/build.py deb
-    - name: Upload deb artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.deb
-        if-no-files-found: error
+    - name: Build
+      run: ./scripts/build_posix.sh
     - name: Publish deb
       if: github.event_name == 'release'
       uses: actions/[email protected]
@@ -37,33 +25,9 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_content_type: binary/octet-stream
-  rpm:
-    runs-on: ubuntu-16.04
-    steps:
-    - uses: actions/[email protected]
-    - name: Set up Python 3.8
-      uses: actions/[email protected]
-      with:
-        python-version: 3.8
-    - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: 2.4
-    - name: Install deps
-      run: |
-        pip install .[all]
-        pip install -r scripts/build-requirements.txt
-        gem install --no-document fpm
-    - name: Build rpm
-      run: python scripts/build.py rpm
-    - name: Upload rpm artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.rpm
-        if-no-files-found: error
     - name: Publish rpm
       if: github.event_name == 'release'
       uses: actions/[email protected]
@@ -71,33 +35,23 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
         asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
         asset_content_type: binary/octet-stream
-  osxpkg:
+  mac:
     runs-on: macos-10.15
     steps:
     - uses: actions/[email protected]
-    - name: Set up Python 3.8
+    - name: Set up Python 3.7
       uses: actions/[email protected]
       with:
-        python-version: 3.8
+        python-version: 3.7
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Install deps
-      run: |
-        pip install .[all]
-        pip install -r scripts/build-requirements.txt
-        gem install --no-document fpm
     - name: Build
-      run: python scripts/build.py osxpkg
-    - name: Upload osxpkg artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.pkg
-        if-no-files-found: error
+      run: ./scripts/build_posix.sh
     - name: Publish
       if: github.event_name == 'release'
       uses: actions/[email protected]
@@ -105,32 +59,20 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
+        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
         asset_name: dvc-${{ github.event.release.tag_name }}.pkg
         asset_content_type: binary/octet-stream
-  exe:
+  windows:
     runs-on: windows-2019
     steps:
     - uses: actions/[email protected]
-    - name: Set up Python 3.8
+    - name: Set up Python 3.7
       uses: actions/[email protected]
       with:
-        python-version: 3.8
-    - name: Install deps
-      run: |
-        pip install .[all]
-        pip install -r scripts/build-requirements.txt
-    - name: Pull images
-      run: |
-        dvc pull
+        python-version: 3.7
     - name: Build
-      run: python scripts\build.py exe
+      run: scripts\build_windows.cmd
       shell: cmd
-    - name: Upload exe artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/innosetup/dvc*.exe
-        if-no-files-found: error
     - name: Publish
       if: github.event_name == 'release'
       env:
@@ -138,10 +80,10 @@ jobs:
       uses: actions/[email protected]
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
+        asset_path: dvc-${{ github.event.release.tag_name }}.exe
         asset_name: dvc-${{ github.event.release.tag_name }}.exe
         asset_content_type: binary/octet-stream
-  pip:
+  python:
     runs-on: ubuntu-16.04
     steps:
     - uses: actions/[email protected]
@@ -166,22 +108,22 @@ jobs:
     - name: Build the package
       uses: snapcore/action-build@v1
       id: snapcraft
-#    - name: Test the package
-#      run: |
-#        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
-#        dvc version
-#        sudo snap remove dvc
+    - name: Test the package
+      run: |
+        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
+        dvc version
+        sudo snap remove dvc
     - name: Publish to the edge channel
       uses: snapcore/action-publish@v1
       if: github.event_name == 'push'
       with:
         store_login: ${{ secrets.STORE_LOGIN }}
         snap: ${{ steps.snapcraft.outputs.snap }}
-        release: edge,v2/edge
+        release: edge,v1/edge
     - name: Publish to the stable channel
       uses: snapcore/action-publish@v1
-      if: ${{ github.event_name == 'release' && github.event.release.released }}
+      if: github.event_name == 'release'
       with:
         store_login: ${{ secrets.STORE_LOGIN }}
         snap: ${{ steps.snapcraft.outputs.snap }}
-        release: stable,v2/stable
+        release: stable,v1/stable

.github/workflows/tests.yaml

@@ -28,13 +28,10 @@ jobs:
         python-version: 3.8
     - name: Install requirements
       run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
-    - name: install temporary dependencies
-      run: |
-        pip install git+https://github.com/isidentical/gdrivefs@service-account
-        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
     - name: Check README
       run: python setup.py checkdocs
-    - uses: pre-commit/[email protected]
+    - name: Check formatting
+      run: pre-commit run --all-files
   tests:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -42,10 +39,9 @@ jobs:
       matrix:
         os: [ubuntu-18.04, windows-2019, macos-10.15]
         pyv: ["3.6", "3.7", "3.8", "3.9"]
-# macos 11.0 workers take too long to start up
-#        include:
-#          - os: "macos-11.0"
-#            pyv: "3.9"
+        include:
+          - os: "macos-11.0"
+            pyv: "3.9"
         exclude:
           - os: macos-10.15
             pyv: "3.9"
@@ -56,27 +52,10 @@ jobs:
       uses: actions/[email protected]
       with:
         python-version: ${{ matrix.pyv }}
-    - name: get pip cache dir
-      id: pip-cache-dir
-      run: |
-        echo "::set-output name=dir::$(pip cache dir)"
-    - name: set pip cache
-      id: pip-cache
-      uses: actions/cache@v2
-      with:
-        path: ${{ steps.pip-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
     - name: install
-      if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
         pip install wheel
         pip install -e ".[all,tests]"
-    - name: install temporary dependencies
-      run: |
-        pip install git+https://github.com/isidentical/gdrivefs@service-account
-        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
     - name: setup git
       run: |
         git config --global user.email "[email protected]"
@@ -88,9 +67,20 @@ jobs:
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined
     - name: upload coverage report
       uses: codecov/[email protected]
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
+    - name: Post TAP
+      # Running custom fork with an option, that does not fail CI on error
+      uses: skshetry/flaky-service/packages/action@master
+      if: always() && github.ref == 'refs/heads/master'
+      with:
+        file-path: ${{github.workspace}}/testresults.tap
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        os: ${{runner.os}}
+        matrix: ${{toJson(matrix)}}
+        repo-description: ${{github.event.repository.description}}
+        fail_ci_if_error: false

.pre-commit-config.yaml

@@ -19,12 +19,6 @@ repos:
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
     rev: 3.8.4
-  - hooks:
-      -  id: mypy
-         language_version: python3
-         files: ^dvc/
-    repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
   - repo: local
     hooks:
     - id: pylint

.pylintrc

@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
-extension-pkg-whitelist=pygit2
+extension-pkg-whitelist=
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
@@ -15,13 +15,10 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook="import sys;
-           from pathlib import Path;
-           from pylint.config import find_pylintrc;
-
-           tests_folder = Path(find_pylintrc()).parent / 'tests'
-           sys.path.append(str(tests_folder));
-           "
+init-hook="import sys; \
+           from pathlib import Path; \
+           from pylint.config import find_pylintrc; \
+           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
 
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
@@ -181,17 +178,13 @@ disable=format,
         # we use it to speedup/optimize
         import-outside-toplevel,
         fixme,
-        # we use cause of the exception for error message output
-        # so some of the times we don't use `raise from` to not show the cause
-        # of the error
-        raise-missing-from,
+
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once). See also the "--disable" option for examples.
-enable=c-extension-no-member,
-       no-else-return,
+enable=c-extension-no-member
 
 
 [REPORTS]
@@ -299,7 +292,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=azure
+ignored-modules=
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.

README.rst

@@ -201,6 +201,38 @@ Contributing
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
 details.
 
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+   :alt: 0
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+   :alt: 1
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+   :alt: 2
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+   :alt: 3
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+   :alt: 4
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+   :alt: 5
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+   :alt: 6
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+   :alt: 7
+
 Mailing List
 ============
 
@@ -222,9 +254,6 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
-Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA.
-
-
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png
    :target: https://dvc.org
    :alt: DVC logo

dvc/_debug.py

@@ -1,91 +0,0 @@
-from contextlib import ExitStack, contextmanager
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from argparse import Namespace
-
-
-@contextmanager
-def instrument(html_output=False):
-    """Run a statistical profiler"""
-    try:
-        from pyinstrument import Profiler  # pylint: disable=import-error
-    except ImportError:
-        print("Failed to run profiler, pyinstrument is not installed")
-        yield
-        return
-
-    profiler = Profiler()
-
-    profiler.start()
-    yield
-    profiler.stop()
-
-    if html_output:
-        profiler.open_in_browser()
-        return
-    print(profiler.output_text(unicode=True, color=True))
-
-
-@contextmanager
-def profile(dump_path: str = None):
-    """Run a cprofile"""
-    import cProfile
-
-    prof = cProfile.Profile()
-    prof.enable()
-
-    yield
-
-    prof.disable()
-    if not dump_path:
-        prof.print_stats(sort="cumtime")
-        return
-    prof.dump_stats(dump_path)
-
-
-@contextmanager
-def debug():
-    try:
-        yield
-    except Exception:  # pylint: disable=broad-except
-        try:
-            import ipdb as pdb  # noqa: T100, pylint: disable=import-error
-        except ImportError:
-            import pdb  # noqa: T100
-        pdb.post_mortem()
-
-        raise  # prevent from jumping ahead
-
-
-@contextmanager
-def debugtools(args: "Namespace" = None, **kwargs):
-    kw = vars(args) if args else {}
-    kw.update(kwargs)
-
-    with ExitStack() as stack:
-        if kw.get("pdb"):
-            stack.enter_context(debug())
-        if kw.get("cprofile") or kw.get("cprofile_dump"):
-            stack.enter_context(profile(kw.get("cprofile_dump")))
-        if kw.get("instrument") or kw.get("instrument_open"):
-            stack.enter_context(instrument(kw.get("instrument_open", False)))
-        yield
-
-
-def add_debugging_flags(parser):
-    from argparse import SUPPRESS
-
-    parser.add_argument(
-        "--cprofile", action="store_true", default=False, help=SUPPRESS,
-    )
-    parser.add_argument("--cprofile-dump", help=SUPPRESS)
-    parser.add_argument(
-        "--pdb", action="store_true", default=False, help=SUPPRESS,
-    )
-    parser.add_argument(
-        "--instrument", action="store_true", default=False, help=SUPPRESS,
-    )
-    parser.add_argument(
-        "--instrument-open", action="store_true", default=False, help=SUPPRESS,
-    )

dvc/analytics.py

@@ -1,6 +1,24 @@
 import json
 import logging
 import os
+import platform
+import sys
+import tempfile
+import uuid
+
+import distro
+import requests
+
+from dvc import __version__
+from dvc.config import Config, to_bool
+from dvc.daemon import daemon
+from dvc.exceptions import NotDvcRepoError
+from dvc.lock import Lock, LockError
+from dvc.repo import Repo
+from dvc.scm import SCM, NoSCM
+from dvc.scm.base import SCMError
+from dvc.utils import env2bool, is_binary
+from dvc.utils.fs import makedirs
 
 logger = logging.getLogger(__name__)
 
@@ -17,10 +35,6 @@ def collect_and_send_report(args=None, return_code=None):
     report as a JSON, where the _collector_ generates it and the _sender_
     removes it after sending it.
     """
-    import tempfile
-
-    from dvc.daemon import daemon
-
     report = {}
 
     # Include command execution information on the report only when available.
@@ -36,9 +50,6 @@ def collect_and_send_report(args=None, return_code=None):
 
 
 def is_enabled():
-    from dvc.config import Config, to_bool
-    from dvc.utils import env2bool
-
     if env2bool("DVC_TEST"):
         return False
 
@@ -58,8 +69,6 @@ def send(path):
     `collect_and_send_report`. Sending happens on another process,
     thus, the need of removing such file afterwards.
     """
-    import requests
-
     url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
@@ -77,11 +86,6 @@ def send(path):
 
 
 def _scm_in_use():
-    from dvc.exceptions import NotDvcRepoError
-    from dvc.repo import Repo
-    from dvc.scm import SCM, NoSCM
-    from dvc.scm.base import SCMError
-
     try:
         scm = SCM(root_dir=Repo.find_root())
         return type(scm).__name__
@@ -95,9 +99,6 @@ def _runtime_info():
     """
     Gather information from the environment where DVC runs to fill a report.
     """
-    from dvc import __version__
-    from dvc.utils import is_binary
-
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
@@ -108,11 +109,6 @@ def _runtime_info():
 
 
 def _system_info():
-    import platform
-    import sys
-
-    import distro
-
     system = platform.system()
 
     if system == "Windows":
@@ -151,12 +147,6 @@ def _find_or_create_user_id():
 
     IDs are generated randomly with UUID.
     """
-    import uuid
-
-    from dvc.config import Config
-    from dvc.lock import Lock, LockError
-    from dvc.utils.fs import makedirs
-
     config_dir = Config.get_dir("global")
     fname = os.path.join(config_dir, "user_id")
     lockfile = os.path.join(config_dir, "user_id.lock")

dvc/api.py

@@ -1,9 +1,15 @@
 import os
 from contextlib import _GeneratorContextManager as GCM
+from contextlib import contextmanager
 
 from funcy import reraise
 
-from dvc.exceptions import OutputNotFoundError, PathMissingError
+from dvc.exceptions import (
+    NotDvcRepoError,
+    OutputNotFoundError,
+    PathMissingError,
+)
+from dvc.external_repo import external_repo
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
@@ -19,17 +25,17 @@ def get_url(path, repo=None, rev=None, remote=None):
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
     """
-    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
+    with _make_repo(repo, rev=rev) as _repo:
         path_info = PathInfo(_repo.root_dir) / path
         with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            metadata = _repo.repo_fs.metadata(path_info)
+            metadata = _repo.repo_tree.metadata(path_info)
 
         if not metadata.is_dvc:
             raise OutputNotFoundError(path, repo)
 
         cloud = metadata.repo.cloud
-        md5 = metadata.repo.dvcfs.info(path_info)["md5"]
-        return cloud.get_url_for(remote, checksum=md5)
+        hash_info = _repo.repo_tree.get_hash(path_info)
+        return cloud.get_url_for(remote, checksum=hash_info.value)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin
@@ -72,7 +78,7 @@ class _OpenContextManager(GCM):
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
+    with _make_repo(repo, rev=rev) as _repo:
         with _repo.open_by_relpath(
             path, remote=remote, mode=mode, encoding=encoding
         ) as fd:
@@ -91,6 +97,19 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
         return fd.read()
 
 
+@contextmanager
+def _make_repo(repo_url=None, rev=None):
+    repo_url = repo_url or os.getcwd()
+    if rev is None and os.path.exists(repo_url):
+        try:
+            yield Repo(repo_url, subrepos=True)
+            return
+        except NotDvcRepoError:
+            pass  # fallthrough to external_repo
+    with external_repo(url=repo_url, rev=rev) as repo:
+        yield repo
+
+
 def make_checkpoint():
     """
     Signal DVC to create a checkpoint experiment.
@@ -103,14 +122,14 @@ def make_checkpoint():
     from time import sleep
 
     from dvc.env import DVC_CHECKPOINT, DVC_ROOT
-    from dvc.stage.monitor import CheckpointTask
+    from dvc.stage.run import CHECKPOINT_SIGNAL_FILE
 
     if os.getenv(DVC_CHECKPOINT) is None:
         return
 
     root_dir = os.getenv(DVC_ROOT, Repo.find_root())
     signal_file = os.path.join(
-        root_dir, Repo.DVC_DIR, "tmp", CheckpointTask.SIGNAL_FILE
+        root_dir, Repo.DVC_DIR, "tmp", CHECKPOINT_SIGNAL_FILE
     )
 
     with builtins.open(signal_file, "w") as fobj:
@@ -120,4 +139,4 @@ def make_checkpoint():
         fobj.flush()
         os.fsync(fobj.fileno())
     while os.path.exists(signal_file):
-        sleep(0.1)
+        sleep(1)

dvc/cache/__init__.py

@@ -1,37 +1,40 @@
+"""Manages cache of a DVC repo."""
 from collections import defaultdict
 
-from dvc.scheme import Schemes
+from ..scheme import Schemes
 
 
-def get_odb(fs):
-    from .base import ObjectDB
-    from .gdrive import GDriveObjectDB
-    from .local import LocalObjectDB
-    from .ssh import SSHObjectDB
+def get_cloud_cache(tree):
+    from .base import CloudCache
+    from .local import LocalCache
+    from .ssh import SSHCache
 
-    if fs.scheme == Schemes.LOCAL:
-        return LocalObjectDB(fs)
+    if tree.scheme == Schemes.LOCAL:
+        return LocalCache(tree)
 
-    if fs.scheme == Schemes.SSH:
-        return SSHObjectDB(fs)
+    if tree.scheme == Schemes.SSH:
+        return SSHCache(tree)
 
-    if fs.scheme == Schemes.GDRIVE:
-        return GDriveObjectDB(fs)
-
-    return ObjectDB(fs)
+    return CloudCache(tree)
 
 
-def _get_odb(repo, settings):
-    from dvc.fs import get_cloud_fs
+def _get_cache(repo, settings):
+    from ..tree import get_cloud_tree
 
     if not settings:
         return None
 
-    fs = get_cloud_fs(repo, **settings)
-    return get_odb(fs)
+    tree = get_cloud_tree(repo, **settings)
+    return get_cloud_cache(tree)
 
 
-class ODBManager:
+class Cache:
+    """Class that manages cache locations of a DVC repo.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that this cache belongs to.
+    """
+
     CACHE_DIR = "cache"
     CLOUD_SCHEMES = [
         Schemes.S3,
@@ -44,7 +47,7 @@ class ODBManager:
     def __init__(self, repo):
         self.repo = repo
         self.config = config = repo.config["cache"]
-        self._odb = {}
+        self._cache = {}
 
         local = config.get("local")
 
@@ -53,33 +56,28 @@ class ODBManager:
         elif "dir" not in config:
             settings = None
         else:
-            from dvc.config_schema import LOCAL_COMMON
+            from ..config import LOCAL_COMMON
 
             settings = {"url": config["dir"]}
             for opt in LOCAL_COMMON.keys():
                 if opt in config:
                     settings[str(opt)] = config.get(opt)
 
-        self._odb[Schemes.LOCAL] = _get_odb(repo, settings)
+        self._cache[Schemes.LOCAL] = _get_cache(repo, settings)
 
-    def _init_odb(self, schemes):
-        for scheme in schemes:
+        for scheme in self.CLOUD_SCHEMES:
             remote = self.config.get(scheme)
             settings = {"name": remote} if remote else None
-            self._odb[scheme] = _get_odb(self.repo, settings)
+            self._cache[scheme] = _get_cache(repo, settings)
 
     def __getattr__(self, name):
-        if name not in self._odb and name in self.CLOUD_SCHEMES:
-            self._init_odb([name])
-
         try:
-            return self._odb[name]
+            return self._cache[name]
         except KeyError as exc:
             raise AttributeError from exc
 
     def by_scheme(self):
-        self._init_odb(self.CLOUD_SCHEMES)
-        yield from self._odb.items()
+        yield from self._cache.items()
 
 
 class NamedCacheItem:

dvc/cache/base.py

@@ -0,0 +1,937 @@
+import itertools
+import json
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from copy import copy
+from typing import Optional
+
+from funcy import decorator
+from shortuuid import uuid
+
+import dvc.prompt as prompt
+from dvc.dir_info import DirInfo
+from dvc.exceptions import (
+    CacheLinkError,
+    CheckoutError,
+    ConfirmRemoveError,
+    DvcException,
+)
+from dvc.progress import Tqdm
+from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
+    slow_link_guard,
+)
+
+from ..tree.base import RemoteActionNotImplemented
+
+logger = logging.getLogger(__name__)
+
+
+class DirCacheError(DvcException):
+    def __init__(self, hash_):
+        super().__init__(
+            f"Failed to load dir cache for hash value: '{hash_}'."
+        )
+
+
+@decorator
+def use_state(call):
+    tree = call._args[0].tree  # pylint: disable=protected-access
+    with tree.state:
+        return call()
+
+
+class CloudCache:
+    """Cloud cache class."""
+
+    DEFAULT_CACHE_TYPES = ["copy"]
+    CACHE_MODE: Optional[int] = None
+
+    def __init__(self, tree):
+        self.tree = tree
+        self.repo = tree.repo
+
+        self.cache_types = tree.config.get("type") or copy(
+            self.DEFAULT_CACHE_TYPES
+        )
+        self.cache_type_confirmed = False
+        self._dir_info = {}
+
+    def get_dir_cache(self, hash_info):
+        assert hash_info
+
+        dir_info = self._dir_info.get(hash_info.value)
+        if dir_info:
+            return dir_info
+
+        try:
+            dir_info = self.load_dir_cache(hash_info)
+        except DirCacheError:
+            dir_info = DirInfo()
+
+        self._dir_info[hash_info.value] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, hash_info):
+        path_info = self.tree.hash_to_path_info(hash_info.value)
+
+        try:
+            with self.tree.open(path_info, "r") as fobj:
+                d = json.load(fobj)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(hash_info) from exc
+
+        if not isinstance(d, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                path_info,
+            )
+            d = []
+
+        return DirInfo.from_list(d)
+
+    def changed(self, path_info, hash_info):
+        """Checks if data has changed.
+
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
+            - There's no file in the cache
+
+        Args:
+            path_info: dict with path information.
+            hash: expected hash value for this data.
+
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug(
+            "checking if '%s'('%s') has changed.", path_info, hash_info
+        )
+
+        if not self.tree.exists(path_info):
+            logger.debug("'%s' doesn't exist.", path_info)
+            return True
+
+        if not hash_info:
+            logger.debug("hash value for '%s' is missing.", path_info)
+            return True
+
+        if self.changed_cache(hash_info):
+            logger.debug(
+                "cache for '%s'('%s') has changed.", path_info, hash_info
+            )
+            return True
+
+        actual = self.tree.get_hash(path_info)
+        if hash_info != actual:
+            logger.debug(
+                "hash value '%s' for '%s' has changed (actual '%s').",
+                hash_info,
+                actual,
+                path_info,
+            )
+            return True
+
+        logger.debug("'%s' hasn't changed.", path_info)
+        return False
+
+    def link(self, from_info, to_info):
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.tree.isfile(from_info)
+
+        self.tree.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self.tree, f"is_{link_type}", None)
+        if is_link and not is_link(path_info):
+            self.tree.remove(path_info)
+            raise DvcException(f"failed to verify {link_type}")
+
+        self.cache_type_confirmed = True
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self.tree, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self._verify_link(to_info, link_types[0])
+                return
+
+            except DvcException as exc:
+                logger.debug(
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
+                )
+                del link_types[0]
+
+        raise CacheLinkError([to_info])
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.tree.exists(to_info):
+            raise DvcException(f"Link '{to_info}' already exists!")
+
+        link_method(from_info, to_info)
+
+        logger.debug(
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
+        )
+
+    def _save_file(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        assert hash_info
+
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
+        if tree == self.tree:
+            if self.changed_cache(hash_info):
+                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.link(cache_info, path_info)
+            elif self.tree.iscopy(path_info) and self._cache_is_copy(
+                path_info
+            ):
+                # Default relink procedure involves unneeded copy
+                self.tree.unprotect(path_info)
+            else:
+                self.tree.remove(path_info)
+                self.link(cache_info, path_info)
+
+            if save_link:
+                self.tree.state.save_link(path_info)
+            # we need to update path and cache, since in case of reflink,
+            # or copy cache type moving original file results in updates on
+            # next executed command, which causes md5 recalculation
+            self.tree.state.save(path_info, hash_info)
+        else:
+            if self.changed_cache(hash_info):
+                with tree.open(path_info, mode="rb") as fobj:
+                    # if tree has fetch enabled, DVC out will be fetched on
+                    # open and we do not need to read/copy any data
+                    if not (
+                        tree.isdvc(path_info, strict=False) and tree.fetch
+                    ):
+                        self.tree.copy_fobj(fobj, cache_info)
+                callback = kwargs.get("download_callback")
+                if callback:
+                    callback(1)
+
+        self.tree.state.save(cache_info, hash_info)
+
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.tree.path_info / ".cache_type_test_file"
+        if not self.tree.exists(test_cache_file):
+            with self.tree.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.tree.remove(workspace_file)
+            self.tree.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
+    def _get_dir_info_hash(self, dir_info):
+        import tempfile
+
+        from dvc.path_info import PathInfo
+        from dvc.utils import tmp_fname
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info.to_list(), fobj, sort_keys=True)
+
+        from_info = PathInfo(tmp)
+        to_info = self.tree.path_info / tmp_fname("")
+        self.tree.upload(from_info, to_info, no_progress_bar=True)
+
+        hash_info = self.tree.get_file_hash(to_info)
+        hash_info.value += self.tree.CHECKSUM_DIR_SUFFIX
+        hash_info.dir_info = dir_info
+        hash_info.nfiles = dir_info.nfiles
+
+        return hash_info, to_info
+
+    @use_state
+    def save_dir_info(self, dir_info, hash_info=None):
+        if (
+            hash_info
+            and hash_info.name == self.tree.PARAM_CHECKSUM
+            and not self.changed_cache_file(hash_info)
+        ):
+            return hash_info
+
+        hi, tmp_info = self._get_dir_info_hash(dir_info)
+        new_info = self.tree.hash_to_path_info(hi.value)
+        if self.changed_cache_file(hi):
+            self.tree.makedirs(new_info.parent)
+            self.tree.move(tmp_info, new_info, mode=self.CACHE_MODE)
+
+        self.tree.state.save(new_info, hi)
+
+        return hi
+
+    def _save_dir(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if not hash_info.dir_info:
+            hash_info.dir_info = tree.cache.get_dir_cache(hash_info)
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
+        for entry_info, entry_hash in Tqdm(
+            hi.dir_info.items(path_info),
+            desc="Saving " + path_info.name,
+            unit="file",
+        ):
+            self._save_file(
+                entry_info, tree, entry_hash, save_link=False, **kwargs
+            )
+
+        if save_link:
+            self.tree.state.save_link(path_info)
+        if self.tree.exists(path_info):
+            self.tree.state.save(path_info, hi)
+
+        cache_info = self.tree.hash_to_path_info(hi.value)
+        self.tree.state.save(cache_info, hi)
+
+    @use_state
+    def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if path_info.scheme != self.tree.scheme:
+            raise RemoteActionNotImplemented(
+                f"save {path_info.scheme} -> {self.tree.scheme}",
+                self.tree.scheme,
+            )
+
+        self._save(path_info, tree, hash_info, save_link, **kwargs)
+
+    def _save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        to_info = self.tree.hash_to_path_info(hash_info.value)
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
+
+        if tree.isdir(path_info):
+            self._save_dir(path_info, tree, hash_info, save_link, **kwargs)
+        else:
+            self._save_file(path_info, tree, hash_info, save_link, **kwargs)
+
+    # Override to return path as a string instead of PathInfo for clouds
+    # which support string paths (see local)
+    def hash_to_path(self, hash_):
+        return self.tree.hash_to_path_info(hash_)
+
+    def changed_cache_file(self, hash_info):
+        """Compare the given hash with the (corresponding) actual one.
+
+        - Use `State` as a cache for computed hashes
+            + The entries are invalidated by taking into account the following:
+                * mtime
+                * inode
+                * size
+                * hash
+
+        - Remove the file from cache if it doesn't match the actual hash
+        """
+        # Prefer string path over PathInfo when possible due to performance
+        cache_info = self.hash_to_path(hash_info.value)
+        if self.tree.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
+        actual = self.tree.get_hash(cache_info)
+
+        logger.debug(
+            "cache '%s' expected '%s' actual '%s'",
+            cache_info,
+            hash_info,
+            actual,
+        )
+
+        if not hash_info or not actual:
+            return True
+
+        if actual.value.split(".")[0] == hash_info.value.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.tree.protect(cache_info)
+            return False
+
+        if self.tree.exists(cache_info):
+            logger.warning("corrupted cache file '%s'.", cache_info)
+            self.tree.remove(cache_info)
+
+        return True
+
+    def _changed_dir_cache(self, hash_info, path_info=None, filter_info=None):
+        if self.changed_cache_file(hash_info):
+            return True
+
+        dir_info = self.get_dir_cache(hash_info)
+        for entry_info, entry_hash in dir_info.items(path_info):
+            if path_info and filter_info:
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
+            if self.changed_cache_file(entry_hash):
+                return True
+
+        return False
+
+    def changed_cache(self, hash_info, path_info=None, filter_info=None):
+        if hash_info.isdir:
+            return self._changed_dir_cache(
+                hash_info, path_info=path_info, filter_info=filter_info
+            )
+        return self.changed_cache_file(hash_info)
+
+    def already_cached(self, path_info):
+        current = self.tree.get_hash(path_info)
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.tree.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                " Are you sure you want to proceed?".format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise ConfirmRemoveError(str(path_info))
+
+        self.tree.remove(path_info)
+
+    def _checkout_file(
+        self, path_info, hash_info, force, progress_callback=None, relink=False
+    ):
+        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
+        if self.tree.exists(path_info):
+            logger.debug("data '%s' will be replaced.", path_info)
+            self.safe_remove(path_info, force=force)
+            added, modified = False, True
+
+        self.link(cache_info, path_info)
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_info)
+        if progress_callback:
+            progress_callback(str(path_info))
+
+        return added, modified and not relink
+
+    def _checkout_dir(
+        self,
+        path_info,
+        hash_info,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        added, modified = False, False
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.tree.exists(path_info):
+            added = True
+            self.tree.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(hash_info)
+
+        logger.debug("Linking directory '%s'.", path_info)
+
+        for entry_info, entry_hash_info in dir_info.items(path_info):
+            entry_cache_info = self.tree.hash_to_path_info(
+                entry_hash_info.value
+            )
+
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
+            if relink or self.changed(entry_info, entry_hash_info):
+                modified = True
+                self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.tree.state.save(entry_info, entry_hash_info)
+            if progress_callback:
+                progress_callback(str(entry_info))
+
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )
+
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_info)
+
+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(self.tree.walk_files(path_info))
+
+        needed_files = {info for info, _ in dir_info.items(path_info)}
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
+            self.safe_remove(path, force)
+
+        return bool(redundant_files)
+
+    @use_state
+    def checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+        quiet=False,
+    ):
+        if path_info.scheme not in ["local", self.tree.scheme]:
+            raise NotImplementedError
+
+        failed = None
+        skip = False
+        if not hash_info:
+            if not quiet:
+                logger.warning(
+                    "No file hash info found for '%s'. It won't be created.",
+                    path_info,
+                )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        elif not relink and not self.changed(path_info, hash_info):
+            logger.debug("Data '%s' didn't change.", path_info)
+            skip = True
+
+        elif self.changed_cache(
+            hash_info, path_info=path_info, filter_info=filter_info
+        ):
+            if not quiet:
+                logger.warning(
+                    "Cache '%s' not found. File '%s' won't be created.",
+                    hash_info,
+                    path_info,
+                )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        if failed or skip:
+            if progress_callback:
+                progress_callback(
+                    str(path_info),
+                    self.get_files_number(
+                        self.tree.path_info, hash_info, filter_info
+                    ),
+                )
+            if failed:
+                raise CheckoutError([failed])
+            return
+
+        logger.debug(
+            "Checking out '%s' with cache '%s'.", path_info, hash_info
+        )
+
+        return self._checkout(
+            path_info,
+            hash_info,
+            force,
+            progress_callback,
+            relink,
+            filter_info,
+        )
+
+    def _checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if not hash_info.isdir:
+            return self._checkout_file(
+                path_info, hash_info, force, progress_callback, relink
+            )
+
+        return self._checkout_dir(
+            path_info, hash_info, force, progress_callback, relink, filter_info
+        )
+
+    def get_files_number(self, path_info, hash_info, filter_info):
+        from funcy.py3 import ilen
+
+        if not hash_info:
+            return 0
+
+        if not hash_info.isdir:
+            return 1
+
+        if not filter_info:
+            return self.get_dir_cache(hash_info).nfiles
+
+        return ilen(
+            filter_info.isin_or_eq(path_info / relpath)
+            for relpath, _ in self.get_dir_cache(hash_info).items()
+        )
+
+    def _get_dir_size(self, dir_info):
+        try:
+            return sum(
+                self.tree.getsize(self.tree.hash_to_path_info(hi.value))
+                for _, hi in dir_info.items()
+            )
+        except FileNotFoundError:
+            return None
+
+    def merge(self, ancestor_info, our_info, their_info):
+        assert our_info
+        assert their_info
+
+        if ancestor_info:
+            ancestor = self.get_dir_cache(ancestor_info)
+        else:
+            ancestor = DirInfo()
+
+        our = self.get_dir_cache(our_info)
+        their = self.get_dir_cache(their_info)
+
+        merged = our.merge(ancestor, their)
+        hash_info = self.save_dir_info(merged)
+        hash_info.size = self._get_dir_size(merged)
+        return hash_info
+
+    @use_state
+    def get_hash(self, tree, path_info):
+        hash_info = tree.get_hash(path_info)
+        if not hash_info.isdir:
+            assert hash_info.name == self.tree.PARAM_CHECKSUM
+            return hash_info
+
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
+        hi.size = hash_info.size
+        return hi
+
+    def set_dir_info(self, hash_info):
+        assert hash_info.isdir
+
+        hash_info.dir_info = self.get_dir_cache(hash_info)
+        hash_info.nfiles = hash_info.dir_info.nfiles
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            if len(prefix) > 2:
+                path_info = self.tree.path_info / prefix[:2] / prefix[2:]
+            else:
+                path_info = self.tree.path_info / prefix[:2]
+            prefix = True
+        else:
+            path_info = self.tree.path_info
+            prefix = False
+        if progress_callback:
+            for file_info in self.tree.walk_files(path_info, prefix=prefix):
+                progress_callback()
+                yield file_info.path
+        else:
+            yield from self.tree.walk_files(path_info, prefix=prefix)
+
+    def _path_to_hash(self, path):
+        parts = self.tree.PATH_CLS(path).parts[-2:]
+
+        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
+            raise ValueError(f"Bad cache file path '{path}'")
+
+        return "".join(parts)
+
+    def list_hashes(self, prefix=None, progress_callback=None):
+        """Iterate over hashes in this tree.
+
+        If `prefix` is specified, only hashes which begin with `prefix`
+        will be returned.
+        """
+        for path in self._list_paths(prefix, progress_callback):
+            try:
+                yield self._path_to_hash(path)
+            except ValueError:
+                logger.debug(
+                    "'%s' doesn't look like a cache file, skipping", path
+                )
+
+    def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
+        count = 0
+        for hash_ in self.list_hashes(prefix, progress_callback):
+            yield hash_
+            count += 1
+            if count > limit:
+                logger.debug(
+                    "`list_hashes()` returned max '{}' hashes, "
+                    "skipping remaining results".format(limit)
+                )
+                return
+
+    def _max_estimation_size(self, hashes):
+        # Max remote size allowed for us to use traverse method
+        return max(
+            self.tree.TRAVERSE_THRESHOLD_SIZE,
+            len(hashes)
+            / self.tree.TRAVERSE_WEIGHT_MULTIPLIER
+            * self.tree.LIST_OBJECT_PAGE_SIZE,
+        )
+
+    def _estimate_remote_size(self, hashes=None, name=None):
+        """Estimate tree size based on number of entries beginning with
+        "00..." prefix.
+        """
+        prefix = "0" * self.tree.TRAVERSE_PREFIX_LEN
+        total_prefixes = pow(16, self.tree.TRAVERSE_PREFIX_LEN)
+        if hashes:
+            max_hashes = self._max_estimation_size(hashes)
+        else:
+            max_hashes = None
+
+        with Tqdm(
+            desc="Estimating size of "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            unit="file",
+        ) as pbar:
+
+            def update(n=1):
+                pbar.update(n * total_prefixes)
+
+            if max_hashes:
+                hashes = self._hashes_with_limit(
+                    max_hashes / total_prefixes, prefix, update
+                )
+            else:
+                hashes = self.list_hashes(prefix, update)
+
+            remote_hashes = set(hashes)
+            if remote_hashes:
+                remote_size = total_prefixes * len(remote_hashes)
+            else:
+                remote_size = total_prefixes
+            logger.debug(f"Estimated remote size: {remote_size} files")
+        return remote_size, remote_hashes
+
+    def list_hashes_traverse(
+        self, remote_size, remote_hashes, jobs=None, name=None
+    ):
+        """Iterate over all hashes found in this tree.
+        Hashes are fetched in parallel according to prefix, except in
+        cases where the remote size is very small.
+
+        All hashes from the remote (including any from the size
+        estimation step passed via the `remote_hashes` argument) will be
+        returned.
+
+        NOTE: For large remotes the list of hashes will be very
+        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
+        and we don't really need all of it at the same time, so it makes
+        sense to use a generator to gradually iterate over it, without
+        keeping all of it in memory.
+        """
+        num_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
+        if num_pages < 256 / self.tree.JOBS:
+            # Fetching prefixes in parallel requires at least 255 more
+            # requests, for small enough remotes it will be faster to fetch
+            # entire cache without splitting it into prefixes.
+            #
+            # NOTE: this ends up re-fetching hashes that were already
+            # fetched during remote size estimation
+            traverse_prefixes = [None]
+            initial = 0
+        else:
+            yield from remote_hashes
+            initial = len(remote_hashes)
+            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
+            if self.tree.TRAVERSE_PREFIX_LEN > 2:
+                traverse_prefixes += [
+                    "{0:0{1}x}".format(i, self.tree.TRAVERSE_PREFIX_LEN)
+                    for i in range(
+                        1, pow(16, self.tree.TRAVERSE_PREFIX_LEN - 2)
+                    )
+                ]
+        with Tqdm(
+            desc="Querying "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            total=remote_size,
+            initial=initial,
+            unit="file",
+        ) as pbar:
+
+            def list_with_update(prefix):
+                return list(
+                    self.list_hashes(
+                        prefix=prefix, progress_callback=pbar.update
+                    )
+                )
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                in_remote = executor.map(list_with_update, traverse_prefixes,)
+                yield from itertools.chain.from_iterable(in_remote)
+
+    def all(self, jobs=None, name=None):
+        """Iterate over all hashes in this tree.
+
+        Hashes will be fetched in parallel threads according to prefix
+        (except for small remotes) and a progress bar will be displayed.
+        """
+        logger.debug(
+            "Fetching all hashes from '{}'".format(
+                name if name else "remote cache"
+            )
+        )
+
+        if not self.tree.CAN_TRAVERSE:
+            return self.list_hashes()
+
+        remote_size, remote_hashes = self._estimate_remote_size(name=name)
+        return self.list_hashes_traverse(
+            remote_size, remote_hashes, jobs, name
+        )
+
+    def _remove_unpacked_dir(self, hash_):
+        pass
+
+    def gc(self, used, jobs=None):
+        removed = False
+        # hashes must be sorted to ensure we always remove .dir files first
+        for hash_ in sorted(
+            self.all(jobs, str(self.tree.path_info)),
+            key=self.tree.is_dir_hash,
+            reverse=True,
+        ):
+            if hash_ in used:
+                continue
+            path_info = self.tree.hash_to_path_info(hash_)
+            if self.tree.is_dir_hash(hash_):
+                # backward compatibility
+                # pylint: disable=protected-access
+                self._remove_unpacked_dir(hash_)
+            self.tree.remove(path_info)
+            removed = True
+
+        return removed
+
+    def list_hashes_exists(self, hashes, jobs=None, name=None):
+        """Return list of the specified hashes which exist in this tree.
+        Hashes will be queried individually.
+        """
+        logger.debug(
+            "Querying {} hashes via object_exists".format(len(hashes))
+        )
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(path_info):
+                ret = self.tree.exists(path_info)
+                pbar.update_msg(str(path_info))
+                return ret
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                path_infos = map(self.tree.hash_to_path_info, hashes)
+                in_remote = executor.map(exists_with_progress, path_infos)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret
+
+    def hashes_exist(self, hashes, jobs=None, name=None):
+        """Check if the given hashes are stored in the remote.
+
+        There are two ways of performing this check:
+
+        - Traverse method: Get a list of all the files in the remote
+            (traversing the cache directory) and compare it with
+            the given hashes. Cache entries will be retrieved in parallel
+            threads according to prefix (i.e. entries starting with, "00...",
+            "01...", and so on) and a progress bar will be displayed.
+
+        - Exists method: For each given hash, run the `exists`
+            method and filter the hashes that aren't on the remote.
+            This is done in parallel threads.
+            It also shows a progress bar when performing the check.
+
+        The reason for such an odd logic is that most of the remotes
+        take much shorter time to just retrieve everything they have under
+        a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
+        check if particular file exists much quicker, use their own
+        implementation of hashes_exist (see ssh, local).
+
+        Which method to use will be automatically determined after estimating
+        the size of the remote cache, and comparing the estimated size with
+        len(hashes). To estimate the size of the remote cache, we fetch
+        a small subset of cache entries (i.e. entries starting with "00...").
+        Based on the number of entries in that subset, the size of the full
+        cache can be estimated, since the cache is evenly distributed according
+        to hash.
+
+        Returns:
+            A list with hashes that were found in the remote
+        """
+        # Remotes which do not use traverse prefix should override
+        # hashes_exist() (see ssh, local)
+        assert self.tree.TRAVERSE_PREFIX_LEN >= 2
+
+        hashes = set(hashes)
+        if len(hashes) == 1 or not self.tree.CAN_TRAVERSE:
+            remote_hashes = self.list_hashes_exists(hashes, jobs, name)
+            return remote_hashes
+
+        # Max remote size allowed for us to use traverse method
+        remote_size, remote_hashes = self._estimate_remote_size(hashes, name)
+
+        traverse_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
+        # For sufficiently large remotes, traverse must be weighted to account
+        # for performance overhead from large lists/sets.
+        # From testing with S3, for remotes with 1M+ files, object_exists is
+        # faster until len(hashes) is at least 10k~100k
+        if remote_size > self.tree.TRAVERSE_THRESHOLD_SIZE:
+            traverse_weight = (
+                traverse_pages * self.tree.TRAVERSE_WEIGHT_MULTIPLIER
+            )
+        else:
+            traverse_weight = traverse_pages
+        if len(hashes) < traverse_weight:
+            logger.debug(
+                "Large remote ('{}' hashes < '{}' traverse weight), "
+                "using object_exists for remaining hashes".format(
+                    len(hashes), traverse_weight
+                )
+            )
+            return list(hashes & remote_hashes) + self.list_hashes_exists(
+                hashes - remote_hashes, jobs, name
+            )
+
+        logger.debug("Querying '{}' hashes via traverse".format(len(hashes)))
+        remote_hashes = set(
+            self.list_hashes_traverse(remote_size, remote_hashes, jobs, name)
+        )
+        return list(hashes & set(remote_hashes))

dvc/cache/local.py

@@ -0,0 +1,91 @@
+import logging
+import os
+
+from funcy import cached_property
+
+from dvc.hash_info import HashInfo
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+
+from ..tree.local import LocalTree
+from ..utils.fs import walk_files
+from .base import CloudCache
+
+logger = logging.getLogger(__name__)
+
+
+class LocalCache(CloudCache):
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+    CACHE_MODE = LocalTree.CACHE_MODE
+    UNPACKED_DIR_SUFFIX = ".unpacked"
+
+    def __init__(self, tree):
+        super().__init__(tree)
+        self.cache_dir = tree.config.get("url")
+
+    @property
+    def cache_dir(self):
+        return self.tree.path_info.fspath if self.tree.path_info else None
+
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.tree.path_info = PathInfo(value) if value else None
+
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def hash_to_path(self, hash_):
+        # NOTE: `self.cache_path` is already normalized so we can simply use
+        # `os.sep` instead of `os.path.join`. This results in this helper
+        # being ~5.5 times faster.
+        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
+
+    def hashes_exist(
+        self, hashes, jobs=None, name=None
+    ):  # pylint: disable=unused-argument
+        return [
+            hash_
+            for hash_ in Tqdm(
+                hashes,
+                unit="file",
+                desc="Querying "
+                + ("cache in " + name if name else "local cache"),
+            )
+            if not self.changed_cache_file(
+                HashInfo(self.tree.PARAM_CHECKSUM, hash_)
+            )
+        ]
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        return super().already_cached(path_info)
+
+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        assert self.tree.path_info is not None
+        if prefix:
+            path_info = self.tree.path_info / prefix[:2]
+            if not self.tree.exists(path_info):
+                return
+        else:
+            path_info = self.tree.path_info
+        # NOTE: use utils.fs walk_files since tree.walk_files will not follow
+        # symlinks
+        if progress_callback:
+            for path in walk_files(path_info):
+                progress_callback()
+                yield path
+        else:
+            yield from walk_files(path_info)
+
+    def _remove_unpacked_dir(self, hash_):
+        info = self.tree.hash_to_path_info(hash_)
+        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
+        self.tree.remove(path_info)

dvc/cache/ssh.py

@@ -7,12 +7,12 @@ from concurrent.futures import ThreadPoolExecutor
 from dvc.progress import Tqdm
 from dvc.utils import to_chunks
 
-from .base import ObjectDB
+from .base import CloudCache
 
 logger = logging.getLogger(__name__)
 
 
-class SSHObjectDB(ObjectDB):
+class SSHCache(CloudCache):
     def batch_exists(self, path_infos, callback):
         def _exists(chunk_and_channel):
             chunk, channel = chunk_and_channel
@@ -28,7 +28,7 @@ class SSHObjectDB(ObjectDB):
                 callback(path)
             return ret
 
-        with self.fs.ssh(path_infos[0]) as ssh:
+        with self.tree.ssh(path_infos[0]) as ssh:
             channels = ssh.open_max_sftp_channels()
             max_workers = len(channels)
 
@@ -47,11 +47,11 @@ class SSHObjectDB(ObjectDB):
         faster than current approach (relying on exists(path_info)) applied in
         remote/base.
         """
-        if not self.fs.CAN_TRAVERSE:
+        if not self.tree.CAN_TRAVERSE:
             return list(set(hashes) & set(self.all()))
 
         # possibly prompt for credentials before "Querying" progress output
-        self.fs.ensure_credentials()
+        self.tree.ensure_credentials()
 
         with Tqdm(
             desc="Querying "
@@ -64,10 +64,10 @@ class SSHObjectDB(ObjectDB):
                 return self.batch_exists(chunks, callback=pbar.update_msg)
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.tree.JOBS
             ) as executor:
-                path_infos = [self.hash_to_path_info(x) for x in hashes]
-                chunks = to_chunks(path_infos, num_chunks=self.fs.JOBS)
+                path_infos = [self.tree.hash_to_path_info(x) for x in hashes]
+                chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
                 results = executor.map(exists_with_progress, chunks)
                 in_remote = itertools.chain.from_iterable(results)
                 ret = list(itertools.compress(hashes, in_remote))
@@ -75,10 +75,10 @@ class SSHObjectDB(ObjectDB):
 
     def _list_paths(self, prefix=None, progress_callback=None):
         if prefix:
-            root = posixpath.join(self.fs.path_info.path, prefix[:2])
+            root = posixpath.join(self.tree.path_info.path, prefix[:2])
         else:
-            root = self.fs.path_info.path
-        with self.fs.ssh(self.fs.path_info) as ssh:
+            root = self.tree.path_info.path
+        with self.tree.ssh(self.tree.path_info) as ssh:
             if prefix and not ssh.exists(root):
                 return
             # If we simply return an iterator then with above closes instantly

dvc/checkout.py

@@ -1,300 +0,0 @@
-import logging
-
-from shortuuid import uuid
-
-import dvc.prompt as prompt
-from dvc.exceptions import (
-    CacheLinkError,
-    CheckoutError,
-    ConfirmRemoveError,
-    DvcException,
-)
-from dvc.objects import check, load
-from dvc.objects.errors import ObjectFormatError
-from dvc.objects.stage import stage
-from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
-    slow_link_guard,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def _changed(path_info, fs, obj, cache):
-    logger.trace("checking if '%s'('%s') has changed.", path_info, obj)
-
-    try:
-        check(cache, obj)
-    except (FileNotFoundError, ObjectFormatError):
-        logger.debug(
-            "cache for '%s'('%s') has changed.", path_info, obj.hash_info
-        )
-        return True
-
-    try:
-        actual = stage(cache, path_info, fs, obj.hash_info.name).hash_info
-    except FileNotFoundError:
-        logger.debug("'%s' doesn't exist.", path_info)
-        return True
-
-    if obj.hash_info != actual:
-        logger.debug(
-            "hash value '%s' for '%s' has changed (actual '%s').",
-            obj.hash_info,
-            actual,
-            path_info,
-        )
-        return True
-
-    logger.trace("'%s' hasn't changed.", path_info)
-    return False
-
-
-def _remove(path_info, fs, cache, force=False):
-    if not fs.exists(path_info):
-        return
-
-    if force:
-        fs.remove(path_info)
-        return
-
-    current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM).hash_info
-    try:
-        obj = load(cache, current)
-        check(cache, obj)
-    except (FileNotFoundError, ObjectFormatError):
-        msg = (
-            f"file/directory '{path_info}' is going to be removed. "
-            "Are you sure you want to proceed?"
-        )
-
-        if not prompt.confirm(msg):
-            raise ConfirmRemoveError(str(path_info))
-
-    fs.remove(path_info)
-
-
-def _verify_link(cache, path_info, link_type):
-    if link_type == "hardlink" and cache.fs.getsize(path_info) == 0:
-        return
-
-    if cache.cache_type_confirmed:
-        return
-
-    is_link = getattr(cache.fs, f"is_{link_type}", None)
-    if is_link and not is_link(path_info):
-        cache.fs.remove(path_info)
-        raise DvcException(f"failed to verify {link_type}")
-
-    cache.cache_type_confirmed = True
-
-
-def _do_link(cache, from_info, to_info, link_method):
-    if cache.fs.exists(to_info):
-        raise DvcException(f"Link '{to_info}' already exists!")
-
-    link_method(from_info, to_info)
-
-    logger.debug(
-        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info,
-    )
-
-
-@slow_link_guard
-def _try_links(cache, from_info, to_info, link_types):
-    while link_types:
-        link_method = getattr(cache.fs, link_types[0])
-        try:
-            _do_link(cache, from_info, to_info, link_method)
-            _verify_link(cache, to_info, link_types[0])
-            return
-
-        except DvcException as exc:
-            logger.debug(
-                "Cache type '%s' is not supported: %s", link_types[0], exc
-            )
-            del link_types[0]
-
-    raise CacheLinkError([to_info])
-
-
-def _link(cache, from_info, to_info):
-    assert cache.fs.isfile(from_info)
-    cache.makedirs(to_info.parent)
-    _try_links(cache, from_info, to_info, cache.cache_types)
-
-
-def _cache_is_copy(cache, path_info):
-    """Checks whether cache uses copies."""
-    if cache.cache_type_confirmed:
-        return cache.cache_types[0] == "copy"
-
-    if set(cache.cache_types) <= {"copy"}:
-        return True
-
-    workspace_file = path_info.with_name("." + uuid())
-    test_cache_file = cache.fs.path_info / ".cache_type_test_file"
-    if not cache.fs.exists(test_cache_file):
-        cache.makedirs(test_cache_file.parent)
-        with cache.fs.open(test_cache_file, "wb") as fobj:
-            fobj.write(bytes(1))
-    try:
-        _link(cache, test_cache_file, workspace_file)
-    finally:
-        cache.fs.remove(workspace_file)
-        cache.fs.remove(test_cache_file)
-
-    cache.cache_type_confirmed = True
-    return cache.cache_types[0] == "copy"
-
-
-def _checkout_file(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
-):
-    """The file is changed we need to checkout a new copy"""
-    modified = False
-    cache_info = cache.hash_to_path_info(obj.hash_info.value)
-    if fs.exists(path_info):
-        if not relink and _changed(path_info, fs, obj, cache):
-            modified = True
-            _remove(path_info, fs, cache, force=force)
-            _link(cache, cache_info, path_info)
-        else:
-            if fs.iscopy(path_info) and _cache_is_copy(cache, path_info):
-                cache.unprotect(path_info)
-            else:
-                _remove(path_info, fs, cache, force=force)
-                _link(cache, cache_info, path_info)
-    else:
-        _link(cache, cache_info, path_info)
-        modified = True
-
-    fs.repo.state.save(path_info, fs, obj.hash_info)
-    if progress_callback:
-        progress_callback(str(path_info))
-
-    return modified
-
-
-def _remove_redundant_files(path_info, fs, obj, cache, force):
-    existing_files = set(fs.walk_files(path_info))
-
-    needed_files = {path_info.joinpath(*key) for key, _ in obj}
-    redundant_files = existing_files - needed_files
-    for path in redundant_files:
-        _remove(path, fs, cache, force)
-
-    return bool(redundant_files)
-
-
-def _checkout_dir(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
-):
-    modified = False, False
-    # Create dir separately so that dir is created
-    # even if there are no files in it
-    if not fs.exists(path_info):
-        modified = True
-        fs.makedirs(path_info)
-
-    logger.debug("Linking directory '%s'.", path_info)
-
-    for entry_key, entry_obj in obj:
-        entry_modified = _checkout_file(
-            path_info.joinpath(*entry_key),
-            fs,
-            entry_obj,
-            cache,
-            force,
-            progress_callback,
-            relink,
-        )
-        if entry_modified:
-            modified = True
-
-    modified = (
-        _remove_redundant_files(path_info, fs, obj, cache, force) or modified
-    )
-
-    fs.repo.state.save(path_info, fs, obj.hash_info)
-
-    # relink is not modified, assume it as nochange
-    return modified and not relink
-
-
-def _checkout(
-    path_info,
-    fs,
-    obj,
-    cache,
-    force=False,
-    progress_callback=None,
-    relink=False,
-):
-    if not obj.hash_info.isdir:
-        ret = _checkout_file(
-            path_info, fs, obj, cache, force, progress_callback, relink
-        )
-    else:
-        ret = _checkout_dir(
-            path_info, fs, obj, cache, force, progress_callback, relink,
-        )
-
-    fs.repo.state.save_link(path_info, fs)
-
-    return ret
-
-
-def checkout(
-    path_info,
-    fs,
-    obj,
-    cache,
-    force=False,
-    progress_callback=None,
-    relink=False,
-    quiet=False,
-):
-    if path_info.scheme not in ["local", cache.fs.scheme]:
-        raise NotImplementedError
-
-    failed = None
-    skip = False
-    if not obj:
-        if not quiet:
-            logger.warning(
-                "No file hash info found for '%s'. It won't be created.",
-                path_info,
-            )
-        _remove(path_info, fs, cache, force=force)
-        failed = path_info
-
-    elif not relink and not _changed(path_info, fs, obj, cache):
-        logger.trace("Data '%s' didn't change.", path_info)
-        skip = True
-    else:
-        try:
-            check(cache, obj)
-        except (FileNotFoundError, ObjectFormatError):
-            if not quiet:
-                logger.warning(
-                    "Cache '%s' not found. File '%s' won't be created.",
-                    obj.hash_info,
-                    path_info,
-                )
-            _remove(path_info, fs, cache, force=force)
-            failed = path_info
-
-    if failed or skip:
-        if progress_callback and obj:
-            progress_callback(
-                str(path_info), len(obj),
-            )
-        if failed:
-            raise CheckoutError([failed])
-        return
-
-    logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
-
-    return _checkout(
-        path_info, fs, obj, cache, force, progress_callback, relink,
-    )

dvc/cli.py

@@ -4,7 +4,6 @@ import logging
 import os
 import sys
 
-from ._debug import add_debugging_flags
 from .command import (
     add,
     cache,
@@ -28,7 +27,6 @@ from .command import (
     imp_url,
     init,
     install,
-    live,
     ls,
     metrics,
     move,
@@ -39,7 +37,6 @@ from .command import (
     repro,
     root,
     run,
-    stage,
     unprotect,
     update,
     version,
@@ -83,10 +80,8 @@ COMMANDS = [
     update,
     git_hook,
     plots,
-    stage,
     experiments,
     check_ignore,
-    live,
 ]
 
 
@@ -142,7 +137,19 @@ def get_parent_parser():
     in order to prevent some weird behavior.
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
-    add_debugging_flags(parent_parser)
+
+    parent_parser.add_argument(
+        "--cprofile",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parent_parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
+
+    parent_parser.add_argument(
+        "--pdb", action="store_true", default=False, help=argparse.SUPPRESS,
+    )
+
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."

dvc/command/add.py

@@ -3,17 +3,13 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
 
 
 class CmdAdd(CmdBase):
     def run(self):
-        from dvc.exceptions import (
-            DvcException,
-            RecursiveAddingWhileUsingFilename,
-        )
-
         try:
             if len(self.args.targets) > 1 and self.args.file:
                 raise RecursiveAddingWhileUsingFilename()
@@ -26,10 +22,6 @@ class CmdAdd(CmdBase):
                 external=self.args.external,
                 glob=self.args.glob,
                 desc=self.args.desc,
-                out=self.args.out,
-                remote=self.args.remote,
-                to_remote=self.args.to_remote,
-                jobs=self.args.jobs,
             )
 
         except DvcException:
@@ -75,38 +67,9 @@ def add_parser(subparsers, parent_parser):
     )
     parser.add_argument(
         "--file",
-        help="Specify name of the .dvc file this command will generate.",
+        help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     )
-    parser.add_argument(
-        "-o",
-        "--out",
-        help="Destination path to put files to.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Download it directly to the remote",
-    )
-    parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to download to",
-        metavar="<name>",
-    )
-    parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
     parser.add_argument(
         "--desc",
         type=str,

dvc/command/base.py

@@ -45,6 +45,14 @@ class CmdBase(ABC):
             updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
             updater.check()
 
+    @property
+    def default_targets(self):
+        """Default targets for `dvc repro`."""
+        from dvc.dvcfile import PIPELINE_FILE
+
+        logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
+        return [PIPELINE_FILE]
+
     @abstractmethod
     def run(self):
         pass

dvc/command/cache.py

@@ -11,21 +11,10 @@ logger = logging.getLogger(__name__)
 class CmdCacheDir(CmdConfig):
     def run(self):
         if self.args.value is None and not self.args.unset:
-            if self.args.level:
-                conf = self.config.read(level=self.args.level)
-            else:
-                # Use merged config with default values
-                conf = self.config
-            self._check(conf, False, "cache", "dir")
-            logger.info(conf["cache"]["dir"])
+            logger.info(self.config["cache"]["dir"])
             return 0
-        with self.config.edit(level=self.args.level) as conf:
-            if self.args.unset:
-                self._check(conf, False, "cache", "dir")
-                del conf["cache"]["dir"]
-            else:
-                self._check(conf, False, "cache")
-                conf["cache"]["dir"] = self.args.value
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
         return 0
 
 

dvc/command/check_ignore.py

@@ -3,6 +3,7 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
@@ -10,7 +11,7 @@ logger = logging.getLogger(__name__)
 class CmdCheckIgnore(CmdBase):
     def __init__(self, args):
         super().__init__(args)
-        self.ignore_filter = self.repo.fs.dvcignore
+        self.ignore_filter = self.repo.tree.dvcignore
 
     def _show_results(self, result):
         if not result.match and not self.args.non_matching:
@@ -54,8 +55,6 @@ class CmdCheckIgnore(CmdBase):
         return ret
 
     def _check_args(self):
-        from dvc.exceptions import DvcException
-
         if not self.args.stdin and not self.args.targets:
             raise DvcException("`targets` or `--stdin` needed")
 

dvc/command/checkout.py

@@ -7,6 +7,7 @@ import colorama
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
+from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
@@ -38,8 +39,6 @@ def log_changes(stats):
 
 class CmdCheckout(CmdBase):
     def run(self):
-        from dvc.utils.humanize import get_summary
-
         stats, exc = None, None
         try:
             stats = self.repo.checkout(

dvc/command/commit.py

@@ -3,14 +3,13 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdCommit(CmdBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         if not self.args.targets:
             self.args.targets = [None]
 
@@ -33,10 +32,7 @@ class CmdCommit(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = (
-        "Record changes to files or directories tracked by DVC"
-        " by storing the current versions in the cache."
-    )
+    COMMIT_HELP = "Save changed data to cache and update DVC-files."
 
     commit_parser = subparsers.add_parser(
         "commit",
@@ -69,7 +65,7 @@ def add_parser(subparsers, parent_parser):
     commit_parser.add_argument(
         "targets",
         nargs="*",
-        help="stages or .dvc files to commit. Optional. "
-        "(Finds all DVC files in the workspace by default.)",
+        help="DVC-files to commit. Optional. "
+        "(Finds all DVC-files in the workspace by default.)",
     ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -8,7 +8,10 @@ from dvc.command.base import CmdBaseNoRepo, append_doc_link
 logger = logging.getLogger(__name__)
 FILE = shtab.FILE
 DIR = shtab.DIRECTORY
-
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
+}
 PREAMBLE = {
     "bash": """
 # $1=COMP_WORDS[1]
@@ -18,73 +21,8 @@ _dvc_compgen_DVCFiles() {
   compgen -f -X '!*Dvcfile' -- $1
   compgen -f -X '!*dvc.yaml' -- $1
 }
-
-_dvc_compgen_stages() {
-    local _dvc_stages=($(dvc stage list -q --names-only))
-    compgen -W "${_dvc_stages[*]}" -- $1
-}
-_dvc_compgen_stages_and_files() {
-    _dvc_compgen_DVCFiles $1
-    _dvc_compgen_stages $1
-}
-
-_dvc_compgen_exps() {
-    local _dvc_exps=($(dvc exp list -q --all --names-only))
-    compgen -W "${_dvc_exps[*]}" -- $1
-}
-    """,
-    "zsh": """
-_dvc_compadd_DVCFiles() {
-    _files -g '(*?.dvc|Dvcfile|dvc.yaml)'
-}
-_dvc_compadd_stages() {
-    # this will also show up the description of the stages
-    _describe 'stages' "($(_dvc_stages_output))"
-}
-
-_dvc_stages_output() {
-  dvc stage list -q | awk '{
-    # escape possible `:` on the stage name
-    sub(/:/, "\\\\\\\\:", $1);
-    # read all of the columns except the first
-    # reading `out` from $2, so as not to have a leading whitespace
-    out=$2; for(i=3;i<=NF;i++){out=out" "$i};
-    # print key, ":" and then single-quote the description
-    # colon is a delimiter used by `_describe` to separate field/description
-    print $1":""\\047"out"\\047"
-    # single quote -> \\047
-    }'
-}
-
-_dvc_compadd_stages_and_files() {
-    _dvc_compadd_DVCFiles
-    _dvc_compadd_stages
-}
-
-_dvc_compadd_exps() {
-    _describe 'experiments' "($(dvc exp list -q -a --names-only))"
-}
-    """,
-}
-
-DVC_FILE = {
-    "bash": "_dvc_compgen_DVCFiles",
-    "zsh": "_dvc_compadd_DVCFiles",
-}
-
-STAGE = {
-    "bash": "_dvc_compgen_stages",
-    "zsh": "_dvc_compadd_stages",
-}
-
-DVCFILES_AND_STAGE = {
-    "bash": "_dvc_compgen_stages_and_files",
-    "zsh": "_dvc_compadd_stages_and_files",
-}
-
-EXPERIMENT = {
-    "bash": "_dvc_compgen_exps",
-    "zsh": "_dvc_compadd_exps",
+""",
+    "zsh": "",
 }
 
 

dvc/command/config.py

@@ -1,115 +1,33 @@
 import argparse
 import logging
-import os
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.config import Config, ConfigError
 
 logger = logging.getLogger(__name__)
 
-NAME_REGEX = r"^(?P<remote>remote\.)?(?P<section>[^\.]*)\.(?P<option>[^\.]*)$"
-
-
-def _name_type(value):
-    import re
-
-    match = re.match(NAME_REGEX, value)
-    if not match:
-        raise argparse.ArgumentTypeError(
-            "name argument should look like "
-            "remote.name.option or "
-            "section.option"
-        )
-    return (
-        bool(match.group("remote")),
-        match.group("section").lower(),
-        match.group("option").lower(),
-    )
-
 
 class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
-        from dvc.config import Config
-
         super().__init__(args)
 
         self.config = Config(validate=False)
 
     def run(self):
-        if self.args.show_origin:
-            if any((self.args.value, self.args.unset)):
-                logger.error(
-                    "--show-origin can't be used together with any of these "
-                    "options: -u/--unset, value"
-                )
-                return 1
-
-        if self.args.list:
-            return self._list()
-
-        if self.args.name is None:
-            logger.error("name argument is required")
-            return 1
-
-        remote, section, opt = self.args.name
+        section, opt = self.args.name.lower().strip().split(".", 1)
 
         if self.args.value is None and not self.args.unset:
-            return self._get(remote, section, opt)
+            conf = self.config.load_one(self.args.level)
+            self._check(conf, section, opt)
+            logger.info(conf[section][opt])
+            return 0
 
-        return self._set(remote, section, opt)
-
-    def _list(self):
-        if any((self.args.name, self.args.value, self.args.unset)):
-            logger.error(
-                "-l/--list can't be used together with any of these "
-                "options: -u/--unset, name, value"
-            )
-            return 1
-
-        levels = [self.args.level] if self.args.level else self.config.LEVELS
-        for level in levels:
-            conf = self.config.read(level)
-            prefix = self._config_file_prefix(
-                self.args.show_origin, self.config, level
-            )
-            logger.info("\n".join(self._format_config(conf, prefix)))
-
-        return 0
-
-    def _get(self, remote, section, opt):
-        from dvc.config import ConfigError
-
-        levels = (
-            [self.args.level] if self.args.level else self.config.LEVELS[::-1]
-        )
-
-        for level in levels:
-            conf = self.config.read(level)
-            if remote:
-                conf = conf["remote"]
-
-            try:
-                self._check(conf, remote, section, opt)
-            except ConfigError:
-                if self.args.level:
-                    raise
-            else:
-                prefix = self._config_file_prefix(
-                    self.args.show_origin, self.config, level
-                )
-                logger.info("{}{}".format(prefix, conf[section][opt]))
-                break
-
-        return 0
-
-    def _set(self, remote, section, opt):
         with self.config.edit(self.args.level) as conf:
-            if remote:
-                conf = conf["remote"]
             if self.args.unset:
-                self._check(conf, remote, section, opt)
+                self._check(conf, section, opt)
                 del conf[section][opt]
             else:
-                self._check(conf, remote, section)
+                self._check(conf, section)
                 conf[section][opt] = self.args.value
 
         if self.args.name == "cache.type":
@@ -121,39 +39,14 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
-    def _check(self, conf, remote, section, opt=None):
-        from dvc.config import ConfigError
-
-        name = "remote" if remote else "section"
+    def _check(self, conf, section, opt=None):
         if section not in conf:
-            raise ConfigError(f"{name} '{section}' doesn't exist")
+            msg = "section {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
 
         if opt and opt not in conf[section]:
-            raise ConfigError(
-                f"option '{opt}' doesn't exist in {name} '{section}'"
-            )
-
-    @staticmethod
-    def _format_config(config, prefix=""):
-        from dvc.utils.flatten import flatten
-
-        for key, value in flatten(config).items():
-            yield f"{prefix}{key}={value}"
-
-    @staticmethod
-    def _config_file_prefix(show_origin, config, level):
-        from dvc.repo import Repo
-
-        if not show_origin:
-            return ""
-
-        level = level or "repo"
-        fname = config.files[level]
-
-        if level in ["local", "repo"]:
-            fname = os.path.relpath(fname, start=Repo.find_root())
-
-        return fname + "\t"
+            msg = "option {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
 
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
@@ -172,21 +65,14 @@ level_group.add_argument(
     const="system",
     help="Use system config.",
 )
-level_group.add_argument(
-    "--project",
-    dest="level",
-    action="store_const",
-    const="repo",
-    help="Use project config (.dvc/config).",
-)
 level_group.add_argument(
     "--local",
     dest="level",
     action="store_const",
     const="local",
-    help="Use local config (.dvc/config.local).",
+    help="Use local config.",
 )
-parent_config_parser.set_defaults(level=None)
+parent_config_parser.set_defaults(level="repo")
 
 
 def add_parser(subparsers, parent_parser):
@@ -206,24 +92,6 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Unset option.",
     )
-    config_parser.add_argument(
-        "name",
-        nargs="?",
-        type=_name_type,
-        help="Option name (section.option or remote.name.option).",
-    )
+    config_parser.add_argument("name", help="Option name.")
     config_parser.add_argument("value", nargs="?", help="Option value.")
-    config_parser.add_argument(
-        "-l",
-        "--list",
-        default=False,
-        action="store_true",
-        help="List all defined config values.",
-    )
-    config_parser.add_argument(
-        "--show-origin",
-        default=False,
-        action="store_true",
-        help="Show the source file containing each config value.",
-    )
     config_parser.set_defaults(func=CmdConfig)

dvc/command/dag.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.stage import Stage
 
 logger = logging.getLogger(__name__)
 
@@ -52,8 +53,6 @@ def _collect_targets(repo, target, outs):
 def _transform(repo, outs):
     import networkx as nx
 
-    from dvc.stage import Stage
-
     def _relabel(node) -> str:
         return node.addressing if isinstance(node, Stage) else str(node)
 

dvc/command/data_sync.py

@@ -3,28 +3,25 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.checkout import log_changes
+from dvc.exceptions import CheckoutError, DvcException
+from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
     def log_summary(self, stats):
-        from dvc.utils.humanize import get_summary
-
         default_msg = "Everything is up to date."
         logger.info(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
     def log_summary(self, stats):
-        from dvc.command.checkout import log_changes
-
         log_changes(stats)
         super().log_summary(stats)
 
     def run(self):
-        from dvc.exceptions import CheckoutError, DvcException
-
         try:
             stats = self.repo.pull(
                 targets=self.args.targets,
@@ -37,7 +34,6 @@ class CmdDataPull(CmdDataBase):
                 force=self.args.force,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
-                glob=self.args.glob,
             )
             self.log_summary(stats)
         except (CheckoutError, DvcException) as exc:
@@ -50,8 +46,6 @@ class CmdDataPull(CmdDataBase):
 
 class CmdDataPush(CmdDataBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             processed_files_count = self.repo.push(
                 targets=self.args.targets,
@@ -63,7 +57,6 @@ class CmdDataPush(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
-                glob=self.args.glob,
             )
             self.log_summary({"pushed": processed_files_count})
         except DvcException:
@@ -74,8 +67,6 @@ class CmdDataPush(CmdDataBase):
 
 class CmdDataFetch(CmdDataBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             processed_files_count = self.repo.fetch(
                 targets=self.args.targets,
@@ -188,12 +179,6 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch run history for all stages.",
     )
-    pull_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Pull cache for targets matching shell-style wildcards.",
-    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
@@ -249,18 +234,10 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Push run history for all stages.",
     )
-    push_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Allows targets containing shell-style wildcards.",
-    )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = (
-        "Download files or directories from remote storage to the cache."
-    )
+    FETCH_HELP = "Download cached files or directories from remote storage."
 
     fetch_parser = subparsers.add_parser(
         "fetch",

dvc/command/destroy.py

@@ -3,14 +3,13 @@ import logging
 
 import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             statement = (
                 "This will destroy all information about your pipelines,"
@@ -33,7 +32,7 @@ class CmdDestroy(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+    DESTROY_HELP = "Remove DVC-files, local DVC config and data cache."
 
     destroy_parser = subparsers.add_parser(
         "destroy",

dvc/command/diff.py

@@ -5,8 +5,8 @@ import os
 
 import colorama
 
-from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +22,7 @@ def _show_md(diff, show_hash=False, hide_missing=False):
 
     header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
-    statuses = ["added", "deleted", "renamed", "modified"]
+    statuses = ["added", "deleted", "modified"]
     if not hide_missing:
         statuses.append("not in cache")
     for status in statuses:
@@ -31,8 +31,6 @@ def _show_md(diff, show_hash=False, hide_missing=False):
             continue
         for entry in entries:
             path = entry["path"]
-            if isinstance(path, dict):
-                path = f"{path['old']} -> {path['new']}"
             if show_hash:
                 check_sum = _digest(entry.get("hash", ""))
                 rows.append([status, check_sum, path])
@@ -74,14 +72,13 @@ class CmdDiff(CmdBase):
             "added": colorama.Fore.GREEN,
             "modified": colorama.Fore.YELLOW,
             "deleted": colorama.Fore.RED,
-            "renamed": colorama.Fore.GREEN,
             "not in cache": colorama.Fore.YELLOW,
         }
 
         summary = {}
         groups = []
 
-        states = ["added", "deleted", "renamed", "modified"]
+        states = ["added", "deleted", "modified"]
         if not hide_missing:
             states.append("not in cache")
         for state in states:
@@ -95,8 +92,6 @@ class CmdDiff(CmdBase):
 
             for entry in entries:
                 path = entry["path"]
-                if isinstance(path, dict):
-                    path = f"{path['old']} -> {path['new']}"
                 checksum = entry.get("hash")
                 summary[state] += 1 if not path.endswith(os.sep) else 0
                 content.append(
@@ -104,7 +99,7 @@ class CmdDiff(CmdBase):
                         space="    ",
                         checksum=_digest(checksum) if checksum else "",
                         separator="  " if checksum else "",
-                        path=path,
+                        path=entry["path"],
                     )
                 )
 
@@ -122,7 +117,7 @@ class CmdDiff(CmdBase):
 
         fmt = (
             "files summary: {added} added, {deleted} deleted,"
-            " {renamed} renamed, {modified} modified"
+            " {modified} modified"
         )
         if not hide_missing:
             fmt += ", {not in cache} not in cache"
@@ -131,24 +126,15 @@ class CmdDiff(CmdBase):
         return "\n\n".join(groups)
 
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
-            diff = self.repo.diff(
-                self.args.a_rev, self.args.b_rev, self.args.targets
-            )
+            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
             show_hash = self.args.show_hash
             hide_missing = self.args.b_rev or self.args.hide_missing
             if hide_missing:
                 del diff["not in cache"]
 
             for key, entries in diff.items():
-                entries = sorted(
-                    entries,
-                    key=lambda entry: entry["path"]["old"]
-                    if isinstance(entry["path"], dict)
-                    else entry["path"],
-                )
+                entries = sorted(entries, key=lambda entry: entry["path"])
                 if not show_hash:
                     for entry in entries:
                         del entry["hash"]
@@ -181,15 +167,6 @@ def add_parser(subparsers, parent_parser):
         help=DIFF_DESCRIPTION,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help=(
-            "Specific DVC-tracked files to compare. "
-            "Accepts one or more file paths."
-        ),
-        metavar="<paths>",
-    ).complete = completion.FILE
     diff_parser.add_argument(
         "a_rev",
         help="Old Git commit to compare (defaults to HEAD)",

dvc/command/experiments.py

@@ -1,64 +1,28 @@
 import argparse
+import io
 import logging
-from collections import Counter, OrderedDict, defaultdict
+import os
+from collections import OrderedDict
 from collections.abc import Mapping
 from datetime import date, datetime
 from itertools import groupby
-from typing import Dict, Iterable, Optional
+from typing import Iterable, Optional
 
 import dvc.prompt as prompt
-from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.command.metrics import DEFAULT_PRECISION
 from dvc.command.repro import CmdRepro
 from dvc.command.repro import add_arguments as add_repro_arguments
 from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.repo.experiments import Experiments
+from dvc.scm.git import Git
 from dvc.utils.flatten import flatten
 
 logger = logging.getLogger(__name__)
 
 
-SHOW_MAX_WIDTH = 1024
-
-
-def _filter_name(names, label, filter_strs):
-    ret = defaultdict(dict)
-    path_filters = defaultdict(list)
-
-    for filter_s in filter_strs:
-        path, _, name = filter_s.rpartition(":")
-        path_filters[path].append(tuple(name.split(".")))
-
-    for path, filters in path_filters.items():
-        if path:
-            match_paths = [path]
-        else:
-            match_paths = names.keys()
-        for length, groups in groupby(filters, len):
-            for group in groups:
-                for match_path in match_paths:
-                    possible_names = [
-                        tuple(name.split(".")) for name in names[match_path]
-                    ]
-                    matches = [
-                        name
-                        for name in possible_names
-                        if name[:length] == group
-                    ]
-                    if not matches:
-                        name = ".".join(group)
-                        raise InvalidArgumentError(
-                            f"'{name}' does not match any known {label}"
-                        )
-                    ret[match_path].update(
-                        {".".join(match): None for match in matches}
-                    )
-
-    return ret
-
-
 def _filter_names(
-    names: Dict[str, Dict[str, None]],
+    names: Iterable,
     label: str,
     include: Optional[Iterable],
     exclude: Optional[Iterable],
@@ -72,32 +36,44 @@ def _filter_names(
                 f" --exclude-{label}"
             )
 
+    names = [tuple(name.split(".")) for name in names]
+
+    def _filter(filters, update_func):
+        filters = [tuple(name.split(".")) for name in filters]
+        for length, groups in groupby(filters, len):
+            for group in groups:
+                matches = [name for name in names if name[:length] == group]
+                if not matches:
+                    name = ".".join(group)
+                    raise InvalidArgumentError(
+                        f"'{name}' does not match any known {label}"
+                    )
+                update_func({match: None for match in matches})
+
     if include:
-        ret = _filter_name(names, label, include)
+        ret: OrderedDict = OrderedDict()
+        _filter(include, ret.update)
     else:
-        ret = names
+        ret = OrderedDict({name: None for name in names})
 
     if exclude:
-        to_remove = _filter_name(names, label, exclude)
-        for path in to_remove:
-            if path in ret:
-                for key in to_remove[path]:
-                    if key in ret[path]:
-                        del ret[path][key]
+        _filter(exclude, ret.difference_update)  # type: ignore[attr-defined]
 
-    return ret
+    return [".".join(name) for name in ret]
 
 
 def _update_names(names, items):
     for name, item in items:
         if isinstance(item, dict):
             item = flatten(item)
-            names[name].update({key: None for key in item})
+            names.update(item.keys())
+        else:
+            names[name] = None
 
 
 def _collect_names(all_experiments, **kwargs):
-    metric_names = defaultdict(dict)
-    param_names = defaultdict(dict)
+    metric_names = set()
+    param_names = set()
 
     for _, experiments in all_experiments.items():
         for exp in experiments.values():
@@ -105,13 +81,13 @@ def _collect_names(all_experiments, **kwargs):
             _update_names(param_names, exp.get("params", {}).items())
 
     metric_names = _filter_names(
-        metric_names,
+        sorted(metric_names),
         "metrics",
         kwargs.get("include_metrics"),
         kwargs.get("exclude_metrics"),
     )
     param_names = _filter_names(
-        (param_names),
+        sorted(param_names),
         "params",
         kwargs.get("include_params"),
         kwargs.get("exclude_params"),
@@ -130,16 +106,15 @@ def _collect_rows(
     sort_by=None,
     sort_order=None,
 ):
-    from dvc.scm.git import Git
-
     if sort_by:
-        sort_path, sort_name, sort_type = _sort_column(
-            sort_by, metric_names, param_names
-        )
+        if sort_by in metric_names:
+            sort_type = "metrics"
+        elif sort_by in param_names:
+            sort_type = "params"
+        else:
+            raise InvalidArgumentError(f"Unknown sort column '{sort_by}'")
         reverse = sort_order == "desc"
-        experiments = _sort_exp(
-            experiments, sort_path, sort_name, sort_type, reverse
-        )
+        experiments = _sort_exp(experiments, sort_by, sort_type, reverse)
 
     new_checkpoint = True
     for i, (rev, exp) in enumerate(experiments.items()):
@@ -195,45 +170,22 @@ def _collect_rows(
         yield row, style
 
 
-def _sort_column(sort_by, metric_names, param_names):
-    path, _, sort_name = sort_by.rpartition(":")
-    matches = set()
-
-    if path:
-        if path in metric_names and sort_name in metric_names[path]:
-            matches.add((path, sort_name, "metrics"))
-        if path in param_names and sort_name in param_names[path]:
-            matches.add((path, sort_name, "params"))
-    else:
-        for path in metric_names:
-            if sort_name in metric_names[path]:
-                matches.add((path, sort_name, "metrics"))
-        for path in param_names:
-            if sort_name in param_names[path]:
-                matches.add((path, sort_name, "params"))
-
-    if len(matches) == 1:
-        return matches.pop()
-    if len(matches) > 1:
-        raise InvalidArgumentError(
-            "Ambiguous sort column '{}' matched '{}'".format(
-                sort_by,
-                ", ".join([f"{path}:{name}" for path, name, _ in matches]),
-            )
-        )
-    raise InvalidArgumentError(f"Unknown sort column '{sort_by}'")
-
-
-def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
+def _sort_exp(experiments, sort_by, typ, reverse):
     def _sort(item):
         rev, exp = item
         tip = exp.get("checkpoint_tip")
         if tip and tip != rev:
             # Sort checkpoint experiments by tip commit
             return _sort((tip, experiments[tip]))
-        data = exp.get(typ, {}).get(sort_path, {})
-        val = flatten(data).get(sort_name)
-        return (val is None, val)
+        for fname, item in exp.get(typ, {}).items():
+            if isinstance(item, dict):
+                item = flatten(item)
+            else:
+                item = {fname: item}
+            if sort_by in item:
+                val = item[sort_by]
+                return (val is None, val)
+        return (True, None)
 
     ret = OrderedDict()
     if "baseline" in experiments:
@@ -257,9 +209,9 @@ def _format_field(val, precision=DEFAULT_PRECISION):
     if isinstance(val, float):
         fmt = f"{{:.{precision}g}}"
         return fmt.format(val)
-    if isinstance(val, Mapping):
+    elif isinstance(val, Mapping):
         return {k: _format_field(v) for k, v in val.items()}
-    if isinstance(val, list):
+    elif isinstance(val, list):
         return [_format_field(x) for x in val]
     return str(val)
 
@@ -276,7 +228,7 @@ def _extend_row(row, names, items, precision):
             item = flatten(item)
         else:
             item = {fname: item}
-        for name in names[fname]:
+        for name in names:
             if name in item:
                 value = item[name]
                 if value is None:
@@ -291,24 +243,24 @@ def _extend_row(row, names, items, precision):
                 row.append("-")
 
 
-def _parse_filter_list(param_list):
+def _parse_list(param_list):
     ret = []
     for param_str in param_list:
-        path, _, param_str = param_str.rpartition(":")
-        if path:
-            ret.extend(f"{path}:{param}" for param in param_str.split(","))
-        else:
-            ret.extend(param_str.split(","))
+        # we don't care about filename prefixes for show, silently
+        # ignore it if provided to keep usage consistent with other
+        # metric/param list command options
+        _, _, param_str = param_str.rpartition(":")
+        ret.extend(param_str.split(","))
     return ret
 
 
-def _experiments_table(all_experiments, **kwargs):
-    from dvc.utils.table import Table
+def _show_experiments(all_experiments, console, **kwargs):
+    from rich.table import Table
 
-    include_metrics = _parse_filter_list(kwargs.pop("include_metrics", []))
-    exclude_metrics = _parse_filter_list(kwargs.pop("exclude_metrics", []))
-    include_params = _parse_filter_list(kwargs.pop("include_params", []))
-    exclude_params = _parse_filter_list(kwargs.pop("exclude_params", []))
+    include_metrics = _parse_list(kwargs.pop("include_metrics", []))
+    exclude_metrics = _parse_list(kwargs.pop("exclude_metrics", []))
+    include_params = _parse_list(kwargs.pop("include_params", []))
+    exclude_params = _parse_list(kwargs.pop("exclude_params", []))
 
     metric_names, param_names = _collect_names(
         all_experiments,
@@ -319,21 +271,13 @@ def _experiments_table(all_experiments, **kwargs):
     )
 
     table = Table()
-    table.add_column(
-        "Experiment", no_wrap=True, header_style="black on grey93"
-    )
+    table.add_column("Experiment", no_wrap=True)
     if not kwargs.get("no_timestamp", False):
-        table.add_column("Created", header_style="black on grey93")
-    _add_data_columns(
-        table,
-        metric_names,
-        justify="right",
-        no_wrap=True,
-        header_style="black on cornsilk1",
-    )
-    _add_data_columns(
-        table, param_names, justify="left", header_style="black on light_cyan1"
-    )
+        table.add_column("Created")
+    for name in metric_names:
+        table.add_column(name, justify="right", no_wrap=True)
+    for name in param_names:
+        table.add_column(name, justify="left")
 
     for base_rev, experiments in all_experiments.items():
         for row, _, in _collect_rows(
@@ -341,20 +285,7 @@ def _experiments_table(all_experiments, **kwargs):
         ):
             table.add_row(*row)
 
-    return table
-
-
-def _add_data_columns(table, names, **kwargs):
-    count = Counter(
-        name for path in names for name in names[path] for path in names
-    )
-    first = True
-    for path in names:
-        for name in names[path]:
-            col_name = name if count[name] == 1 else f"{path}:{name}"
-            kwargs["collapse"] = False if first else True
-            table.add_column(col_name, **kwargs)
-            first = False
+    console.print(table)
 
 
 def _format_json(item):
@@ -367,13 +298,17 @@ class CmdExperimentsShow(CmdBase):
     def run(self):
         from rich.console import Console
 
+        from dvc.utils.pager import pager
+
+        if not self.repo.experiments:
+            return 0
+
         try:
             all_experiments = self.repo.experiments.show(
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
                 all_commits=self.args.all_commits,
                 sha_only=self.args.sha,
-                num=self.args.num,
             )
 
             if self.args.show_json:
@@ -382,13 +317,23 @@ class CmdExperimentsShow(CmdBase):
                 logger.info(json.dumps(all_experiments, default=_format_json))
                 return 0
 
+            if self.args.no_pager:
+                console = Console()
+            else:
+                # Note: rich does not currently include a native way to force
+                # infinite width for use with a pager
+                console = Console(
+                    file=io.StringIO(), force_terminal=True, width=9999
+                )
+
             if self.args.precision is None:
                 precision = DEFAULT_PRECISION
             else:
                 precision = self.args.precision
 
-            table = _experiments_table(
+            _show_experiments(
                 all_experiments,
+                console,
                 include_metrics=self.args.include_metrics,
                 exclude_metrics=self.args.exclude_metrics,
                 include_params=self.args.include_params,
@@ -399,22 +344,8 @@ class CmdExperimentsShow(CmdBase):
                 precision=precision,
             )
 
-            console = Console()
-            if self.args.no_pager:
-                console.print(table)
-            else:
-                from dvc.utils.pager import DvcPager
-
-                # NOTE: rich does not have native support for unlimited width
-                # via pager. we override rich table compression by setting
-                # console width to the full width of the table
-                measurement = table.__rich_measure__(console, SHOW_MAX_WIDTH)
-                console._width = (  # pylint: disable=protected-access
-                    measurement.maximum
-                )
-                with console.pager(pager=DvcPager(), styles=True):
-                    console.print(table)
-
+            if not self.args.no_pager:
+                pager(console.file.getvalue())
         except DvcException:
             logger.exception("failed to show experiments")
             return 1
@@ -424,10 +355,10 @@ class CmdExperimentsShow(CmdBase):
 
 class CmdExperimentsApply(CmdBase):
     def run(self):
+        if not self.repo.experiments:
+            return 0
 
-        self.repo.experiments.apply(
-            self.args.experiment, force=self.args.force
-        )
+        self.repo.experiments.apply(self.args.experiment)
 
         return 0
 
@@ -471,6 +402,8 @@ def _show_diff(
 
 class CmdExperimentsDiff(CmdBase):
     def run(self):
+        if not self.repo.experiments:
+            return 0
 
         try:
             diff = self.repo.experiments.diff(
@@ -512,46 +445,47 @@ class CmdExperimentsDiff(CmdBase):
 
 class CmdExperimentsRun(CmdRepro):
     def run(self):
-        from dvc.command.metrics import _show_metrics
+        if not self.repo.experiments:
+            return 0
 
-        if self.args.reset and self.args.checkpoint_resume:
-            raise InvalidArgumentError(
-                "--reset and --rev are mutually exclusive."
-            )
+        saved_dir = os.path.realpath(os.curdir)
+        os.chdir(self.args.cwd)
 
-        if self.args.reset:
-            logger.info("Any existing checkpoints will be reset and re-run.")
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
 
-        results = self.repo.experiments.run(
-            name=self.args.name,
-            queue=self.args.queue,
-            run_all=self.args.run_all,
-            jobs=self.args.jobs,
-            params=self.args.set_param,
-            checkpoint_resume=self.args.checkpoint_resume,
-            reset=self.args.reset,
-            tmp_dir=self.args.tmp_dir,
-            **self._repro_kwargs,
-        )
+        ret = 0
+        for target in self.args.targets:
+            try:
+                self.repo.experiments.run(
+                    target,
+                    name=self.args.name,
+                    queue=self.args.queue,
+                    run_all=self.args.run_all,
+                    jobs=self.args.jobs,
+                    params=self.args.params,
+                    checkpoint_resume=self.args.checkpoint_resume,
+                    **self._repro_kwargs,
+                )
+            except DvcException:
+                logger.exception("")
+                ret = 1
+                break
 
-        if self.args.metrics and results:
-            metrics = self.repo.metrics.show(revs=list(results))
-            metrics.pop("workspace", None)
-            logger.info(_show_metrics(metrics))
-
-        return 0
-
-
-def _raise_error_if_all_disabled(**kwargs):
-    if not any(kwargs.values()):
-        raise InvalidArgumentError(
-            "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
-            "or `--all-commits` needs to be set."
-        )
+        os.chdir(saved_dir)
+        return ret
 
 
 class CmdExperimentsGC(CmdRepro):
     def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        if not self.repo.experiments:
+            return 0
+
         _raise_error_if_all_disabled(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
@@ -602,110 +536,23 @@ class CmdExperimentsGC(CmdRepro):
 
 class CmdExperimentsBranch(CmdBase):
     def run(self):
+        if not self.repo.experiments:
+            return 0
 
         self.repo.experiments.branch(self.args.experiment, self.args.branch)
 
         return 0
 
 
-class CmdExperimentsList(CmdBase):
-    def run(self):
-        names_only = self.args.names_only
-        exps = self.repo.experiments.ls(
-            rev=self.args.rev,
-            git_remote=self.args.git_remote,
-            all_=self.args.all,
-        )
-        for baseline in exps:
-            tag = self.repo.scm.describe(baseline)
-            if not tag:
-                branch = self.repo.scm.describe(baseline, base="refs/heads")
-                if branch:
-                    tag = branch.split("/")[-1]
-            name = tag if tag else baseline[:7]
-            if not names_only:
-                print(f"{name}:")
-            for exp_name in exps[baseline]:
-                indent = "" if names_only else "\t"
-                print(f"{indent}{exp_name}")
-
-        return 0
-
-
-class CmdExperimentsPush(CmdBase):
-    def run(self):
-
-        self.repo.experiments.push(
-            self.args.git_remote,
-            self.args.experiment,
-            force=self.args.force,
-            push_cache=self.args.push_cache,
-            dvc_remote=self.args.dvc_remote,
-            jobs=self.args.jobs,
-            run_cache=self.args.run_cache,
-        )
-
-        logger.info(
-            "Pushed experiment '%s' to Git remote '%s'.",
-            self.args.experiment,
-            self.args.git_remote,
-        )
-        if not self.args.push_cache:
-            logger.info(
-                "To push cached outputs for this experiment to DVC remote "
-                "storage, re-run this command without '--no-cache'."
-            )
-
-        return 0
-
-
-class CmdExperimentsPull(CmdBase):
-    def run(self):
-
-        self.repo.experiments.pull(
-            self.args.git_remote,
-            self.args.experiment,
-            force=self.args.force,
-            pull_cache=self.args.pull_cache,
-            dvc_remote=self.args.dvc_remote,
-            jobs=self.args.jobs,
-            run_cache=self.args.run_cache,
-        )
-
-        logger.info(
-            "Pulled experiment '%s' from Git remote '%s'. ",
-            self.args.experiment,
-            self.args.git_remote,
-        )
-        if not self.args.pull_cache:
-            logger.info(
-                "To pull cached outputs for this experiment from DVC remote "
-                "storage, re-run this command without '--no-cache'."
-            )
-
-        return 0
-
-
-class CmdExperimentsRemove(CmdBase):
-    def run(self):
-
-        self.repo.experiments.remove(
-            exp_names=self.args.experiment, queue=self.args.queue,
-        )
-
-        return 0
-
-
 def add_parser(subparsers, parent_parser):
-    EXPERIMENTS_HELP = "Commands to run and compare experiments."
+    EXPERIMENTS_HELP = "Commands to display and compare experiments."
 
     experiments_parser = subparsers.add_parser(
         "experiments",
         parents=[parent_parser],
         aliases=["exp"],
-        description=append_doc_link(EXPERIMENTS_HELP, "exp"),
+        description=append_doc_link(EXPERIMENTS_HELP, "experiments"),
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        help=EXPERIMENTS_HELP,
     )
 
     experiments_subparsers = experiments_parser.add_subparsers(
@@ -720,7 +567,7 @@ def add_parser(subparsers, parent_parser):
     experiments_show_parser = experiments_subparsers.add_parser(
         "show",
         parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "exp/show"),
+        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "experiments/show"),
         help=EXPERIMENTS_SHOW_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -729,30 +576,20 @@ def add_parser(subparsers, parent_parser):
         "--all-branches",
         action="store_true",
         default=False,
-        help="Show experiments derived from the tip of all Git branches.",
+        help="Show metrics for all branches.",
     )
     experiments_show_parser.add_argument(
         "-T",
         "--all-tags",
         action="store_true",
         default=False,
-        help="Show experiments derived from all Git tags.",
+        help="Show metrics for all tags.",
     )
     experiments_show_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
-        help="Show experiments derived from all Git commits.",
-    )
-    experiments_show_parser.add_argument(
-        "-n",
-        "--num",
-        type=int,
-        default=1,
-        dest="num",
-        metavar="<num>",
-        help="Show the last `num` commits from HEAD.",
+        help="Show metrics for all commits.",
     )
     experiments_show_parser.add_argument(
         "--no-pager",
@@ -834,19 +671,15 @@ def add_parser(subparsers, parent_parser):
     experiments_apply_parser = experiments_subparsers.add_parser(
         "apply",
         parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_APPLY_HELP, "exp/apply"),
+        description=append_doc_link(
+            EXPERIMENTS_APPLY_HELP, "experiments/apply"
+        ),
         help=EXPERIMENTS_APPLY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    experiments_apply_parser.add_argument(
-        "--no-force",
-        action="store_false",
-        dest="force",
-        help="Fail if this command would overwrite conflicting changes.",
-    )
     experiments_apply_parser.add_argument(
         "experiment", help="Experiment to be applied.",
-    ).complete = completion.EXPERIMENT
+    )
     experiments_apply_parser.set_defaults(func=CmdExperimentsApply)
 
     EXPERIMENTS_DIFF_HELP = (
@@ -855,18 +688,18 @@ def add_parser(subparsers, parent_parser):
     experiments_diff_parser = experiments_subparsers.add_parser(
         "diff",
         parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "exp/diff"),
+        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "experiments/diff"),
         help=EXPERIMENTS_DIFF_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_diff_parser.add_argument(
         "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
-    ).complete = completion.EXPERIMENT
+    )
     experiments_diff_parser.add_argument(
         "b_rev",
         nargs="?",
         help="New experiment to compare (defaults to the current workspace)",
-    ).complete = completion.EXPERIMENT
+    )
     experiments_diff_parser.add_argument(
         "--all",
         action="store_true",
@@ -914,29 +747,42 @@ def add_parser(subparsers, parent_parser):
     experiments_run_parser = experiments_subparsers.add_parser(
         "run",
         parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_RUN_HELP, "exp/run"),
+        description=append_doc_link(EXPERIMENTS_RUN_HELP, "experiments/run"),
         help=EXPERIMENTS_RUN_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     _add_run_common(experiments_run_parser)
     experiments_run_parser.add_argument(
+        "--checkpoint-resume", type=str, default=None, help=argparse.SUPPRESS,
+    )
+    experiments_run_parser.set_defaults(func=CmdExperimentsRun)
+
+    EXPERIMENTS_RESUME_HELP = "Resume checkpoint experiments."
+    experiments_resume_parser = experiments_subparsers.add_parser(
+        "resume",
+        parents=[parent_parser],
+        aliases=["res"],
+        description=append_doc_link(
+            EXPERIMENTS_RESUME_HELP, "experiments/resume"
+        ),
+        help=EXPERIMENTS_RESUME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _add_run_common(experiments_resume_parser)
+    experiments_resume_parser.add_argument(
         "-r",
         "--rev",
         type=str,
+        default=Experiments.LAST_CHECKPOINT,
         dest="checkpoint_resume",
         help=(
             "Continue the specified checkpoint experiment. "
-            "(Only required for explicitly resuming checkpoints in queued "
-            "or temp dir runs.)"
+            "If no experiment revision is provided, "
+            "the most recently run checkpoint experiment will be used."
         ),
         metavar="<experiment_rev>",
-    ).complete = completion.EXPERIMENT
-    experiments_run_parser.add_argument(
-        "--reset",
-        action="store_true",
-        help="Reset existing checkpoints and restart the experiment.",
     )
-    experiments_run_parser.set_defaults(func=CmdExperimentsRun)
+    experiments_resume_parser.set_defaults(func=CmdExperimentsRun)
 
     EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
     EXPERIMENTS_GC_DESCRIPTION = (
@@ -946,7 +792,9 @@ def add_parser(subparsers, parent_parser):
     experiments_gc_parser = experiments_subparsers.add_parser(
         "gc",
         parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_GC_DESCRIPTION, "exp/gc"),
+        description=append_doc_link(
+            EXPERIMENTS_GC_DESCRIPTION, "experiments/gc"
+        ),
         help=EXPERIMENTS_GC_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -999,7 +847,9 @@ def add_parser(subparsers, parent_parser):
     experiments_branch_parser = experiments_subparsers.add_parser(
         "branch",
         parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_BRANCH_HELP, "exp/branch"),
+        description=append_doc_link(
+            EXPERIMENTS_BRANCH_HELP, "experiments/branch"
+        ),
         help=EXPERIMENTS_BRANCH_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -1011,176 +861,6 @@ def add_parser(subparsers, parent_parser):
     )
     experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)
 
-    EXPERIMENTS_LIST_HELP = "List local and remote experiments."
-    experiments_list_parser = experiments_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_LIST_HELP, "exp/list"),
-        help=EXPERIMENTS_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    experiments_list_parser.add_argument(
-        "--rev",
-        type=str,
-        default=None,
-        help=(
-            "List experiments derived from the specified revision. "
-            "Defaults to HEAD if neither `--rev` nor `--all` are specified."
-        ),
-        metavar="<rev>",
-    )
-    experiments_list_parser.add_argument(
-        "--all", action="store_true", help="List all experiments.",
-    )
-    experiments_list_parser.add_argument(
-        "--names-only",
-        action="store_true",
-        help="Only output experiment names (without parent commits).",
-    )
-    experiments_list_parser.add_argument(
-        "git_remote",
-        nargs="?",
-        default=None,
-        help=(
-            "Optional Git remote name or Git URL. If provided, experiments "
-            "from the specified Git repository will be listed instead of "
-            "local experiments."
-        ),
-        metavar="[<git_remote>]",
-    )
-    experiments_list_parser.set_defaults(func=CmdExperimentsList)
-
-    EXPERIMENTS_PUSH_HELP = "Push a local experiment to a Git remote."
-    experiments_push_parser = experiments_subparsers.add_parser(
-        "push",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_PUSH_HELP, "exp/push"),
-        help=EXPERIMENTS_PUSH_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    experiments_push_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="Replace experiment in the Git remote if it already exists.",
-    )
-    experiments_push_parser.add_argument(
-        "--no-cache",
-        action="store_false",
-        dest="push_cache",
-        help=(
-            "Do not push cached outputs for this experiment to DVC remote "
-            "storage."
-        ),
-    )
-    experiments_push_parser.add_argument(
-        "-r",
-        "--remote",
-        dest="dvc_remote",
-        metavar="<name>",
-        help="Name of the DVC remote to use when pushing cached outputs.",
-    )
-    experiments_push_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        metavar="<number>",
-        help=(
-            "Number of jobs to run simultaneously when pushing to DVC remote "
-            "storage."
-        ),
-    )
-    experiments_push_parser.add_argument(
-        "--run-cache",
-        action="store_true",
-        default=False,
-        help="Push run history for all stages.",
-    )
-    experiments_push_parser.add_argument(
-        "git_remote",
-        help="Git remote name or Git URL.",
-        metavar="<git_remote>",
-    )
-    experiments_push_parser.add_argument(
-        "experiment", help="Experiment to push.", metavar="<experiment>",
-    ).complete = completion.EXPERIMENT
-    experiments_push_parser.set_defaults(func=CmdExperimentsPush)
-
-    EXPERIMENTS_PULL_HELP = "Pull an experiment from a Git remote."
-    experiments_pull_parser = experiments_subparsers.add_parser(
-        "pull",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_PULL_HELP, "exp/pull"),
-        help=EXPERIMENTS_PULL_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    experiments_pull_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="Replace local experiment already exists.",
-    )
-    experiments_pull_parser.add_argument(
-        "--no-cache",
-        action="store_false",
-        dest="pull_cache",
-        help=(
-            "Do not pull cached outputs for this experiment from DVC remote "
-            "storage."
-        ),
-    )
-    experiments_pull_parser.add_argument(
-        "-r",
-        "--remote",
-        dest="dvc_remote",
-        metavar="<name>",
-        help="Name of the DVC remote to use when pulling cached outputs.",
-    )
-    experiments_pull_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        metavar="<number>",
-        help=(
-            "Number of jobs to run simultaneously when pulling from DVC "
-            "remote storage."
-        ),
-    )
-    experiments_pull_parser.add_argument(
-        "--run-cache",
-        action="store_true",
-        default=False,
-        help="Pull run history for all stages.",
-    )
-    experiments_pull_parser.add_argument(
-        "git_remote",
-        help="Git remote name or Git URL.",
-        metavar="<git_remote>",
-    )
-    experiments_pull_parser.add_argument(
-        "experiment", help="Experiment to pull.", metavar="<experiment>",
-    )
-    experiments_pull_parser.set_defaults(func=CmdExperimentsPull)
-
-    EXPERIMENTS_REMOVE_HELP = "Remove local experiments."
-    experiments_remove_parser = experiments_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_REMOVE_HELP, "exp/remove"),
-        help=EXPERIMENTS_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    experiments_remove_parser.add_argument(
-        "--queue", action="store_true", help="Remove all queued experiments.",
-    )
-    experiments_remove_parser.add_argument(
-        "experiment",
-        nargs="*",
-        help="Experiments to remove.",
-        metavar="<experiment>",
-    )
-    experiments_remove_parser.set_defaults(func=CmdExperimentsRemove)
-
 
 def _add_run_common(parser):
     """Add common args for 'exp run' and 'exp resume'."""
@@ -1194,15 +874,13 @@ def _add_run_common(parser):
             "Human-readable experiment name. If not specified, a name will "
             "be auto-generated."
         ),
-        metavar="<name>",
     )
     parser.add_argument(
-        "-S",
-        "--set-param",
+        "--params",
         action="append",
         default=[],
-        help="Use the specified param value when reproducing pipelines.",
-        metavar="[<filename>:]<param_name>=<param_value>",
+        help="Use the specified param values when reproducing pipelines.",
+        metavar="[<filename>:]<params_list>",
     )
     parser.add_argument(
         "--queue",
@@ -1214,7 +892,7 @@ def _add_run_common(parser):
         "--run-all",
         action="store_true",
         default=False,
-        help="Execute all experiments in the run queue. Implies --temp.",
+        help="Execute all experiments in the run queue.",
     )
     parser.add_argument(
         "-j",
@@ -1223,12 +901,3 @@ def _add_run_common(parser):
         help="Run the specified number of experiments at a time in parallel.",
         metavar="<number>",
     )
-    parser.add_argument(
-        "--temp",
-        action="store_true",
-        dest="tmp_dir",
-        help=(
-            "Run this experiment in a separate temporary directory instead of "
-            "your workspace."
-        ),
-    )

dvc/command/gc.py

@@ -30,8 +30,6 @@ class CmdGC(CmdBase):
             msg += " and all git branches"
         elif self.args.all_tags:
             msg += " and all git tags"
-        elif self.args.all_experiments:
-            msg += " and all experiments"
 
         if self.args.repos:
             msg += " of the current and the following repos:"
@@ -51,7 +49,6 @@ class CmdGC(CmdBase):
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
             all_commits=self.args.all_commits,
-            all_experiments=self.args.all_experiments,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -102,12 +99,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Keep data files for all Git commits.",
     )
-    gc_parser.add_argument(
-        "--all-experiments",
-        action="store_true",
-        default=False,
-        help="Keep data files for all experiments.",
-    )
     gc_parser.add_argument(
         "-c",
         "--cloud",

dvc/command/get.py

@@ -39,7 +39,6 @@ class CmdGet(CmdBaseNoRepo):
                 path=self.args.path,
                 out=self.args.out,
                 rev=self.args.rev,
-                jobs=self.args.jobs,
             )
             return 0
         except DvcException:
@@ -85,15 +84,4 @@ def add_parser(subparsers, parent_parser):
         help="Print the storage location (URL) the target data would be "
         "downloaded from, and exit.",
     )
-    get_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
     get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -14,7 +14,7 @@ class CmdGetUrl(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            Repo.get_url(self.args.url, out=self.args.out, jobs=self.args.jobs)
+            Repo.get_url(self.args.url, out=self.args.out)
             return 0
         except DvcException:
             logger.exception(f"failed to get '{self.args.url}'")
@@ -36,15 +36,4 @@ def add_parser(subparsers, parent_parser):
     get_parser.add_argument(
         "out", nargs="?", help="Destination path to put data to.",
     ).complete = completion.DIR
-    get_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -69,13 +69,14 @@ class CmdMergeDriver(CmdHookBase):
         dvc = Repo()
 
         try:
-            ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
-            our = Dvcfile(dvc, self.args.our, verify=False)
-            their = Dvcfile(dvc, self.args.their, verify=False)
+            with dvc.state:
+                ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
+                our = Dvcfile(dvc, self.args.our, verify=False)
+                their = Dvcfile(dvc, self.args.their, verify=False)
 
-            our.merge(ancestor, their)
+                our.merge(ancestor, their)
 
-            return 0
+                return 0
         finally:
             dvc.close()
 

dvc/command/imp.py

@@ -19,7 +19,6 @@ class CmdImport(CmdBase):
                 rev=self.args.rev,
                 no_exec=self.args.no_exec,
                 desc=self.args.desc,
-                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception(
@@ -65,14 +64,14 @@ def add_parser(subparsers, parent_parser):
     )
     import_parser.add_argument(
         "--file",
-        help="Specify name of the .dvc file this command will generate.",
+        help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     )
     import_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create .dvc file without actually downloading it.",
+        help="Only create DVC-file without actually downloading it.",
     )
     import_parser.add_argument(
         "--desc",
@@ -83,15 +82,4 @@ def add_parser(subparsers, parent_parser):
             "This doesn't affect any DVC operations."
         ),
     )
-    import_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -16,10 +16,7 @@ class CmdImportUrl(CmdBase):
                 out=self.args.out,
                 fname=self.args.file,
                 no_exec=self.args.no_exec,
-                remote=self.args.remote,
-                to_remote=self.args.to_remote,
                 desc=self.args.desc,
-                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception(
@@ -62,37 +59,14 @@ def add_parser(subparsers, parent_parser):
     ).complete = completion.DIR
     import_parser.add_argument(
         "--file",
-        help="Specify name of the .dvc file this command will generate.",
+        help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     ).complete = completion.DIR
     import_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create .dvc file without actually downloading it.",
-    )
-    import_parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Download it directly to the remote",
-    )
-    import_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to download to",
-        metavar="<name>",
-    )
-    import_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
+        help="Only create DVC-file without actually downloading it.",
     )
     import_parser.add_argument(
         "--desc",

dvc/command/live.py

@@ -1,94 +0,0 @@
-import argparse
-import logging
-import os
-
-from dvc.command import completion
-from dvc.command.base import CmdBase, fix_subparsers
-from dvc.utils.html import write
-
-logger = logging.getLogger(__name__)
-
-
-class CmdLive(CmdBase):
-    UNINITIALIZED = True
-
-    def _run(self, target, revs=None):
-        metrics, plots = self.repo.live.show(target=target, revs=revs)
-
-        html_path = self.args.target + ".html"
-        write(html_path, plots, metrics)
-
-        logger.info(f"\nfile://{os.path.abspath(html_path)}")
-
-        return 0
-
-
-class CmdLiveShow(CmdLive):
-    def run(self):
-        return self._run(self.args.target)
-
-
-class CmdLiveDiff(CmdLive):
-    def run(self):
-        return self._run(self.args.target, self.args.revs)
-
-
-def shared_parent_parser():
-    parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument(
-        "target", help="Logs dir to produce summary from",
-    ).complete = completion.DIR
-    parent_parser.add_argument(
-        "-o",
-        "--out",
-        default=None,
-        help="Destination path to save plots to",
-        metavar="<path>",
-    ).complete = completion.DIR
-    return parent_parser
-
-
-def add_parser(subparsers, parent_parser):
-    LIVE_DESCRIPTION = (
-        "Commands to visualize and compare dvclive-produced logs."
-    )
-    live_parser = subparsers.add_parser(
-        "live",
-        parents=[parent_parser],
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        description=LIVE_DESCRIPTION,
-    )
-    live_subparsers = live_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc live CMD --help` to display command-specific help.",
-    )
-
-    fix_subparsers(live_subparsers)
-
-    SHOW_HELP = "Visualize dvclive directory content."
-    live_show_parser = live_subparsers.add_parser(
-        "show",
-        parents=[parent_parser, shared_parent_parser()],
-        help=SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    live_show_parser.set_defaults(func=CmdLiveShow)
-
-    DIFF_HELP = (
-        "Show multiple versions of dvclive data, "
-        "by plotting it in single view."
-    )
-    live_diff_parser = live_subparsers.add_parser(
-        "diff",
-        parents=[parent_parser, shared_parent_parser()],
-        help=DIFF_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    live_diff_parser.add_argument(
-        "--revs",
-        nargs="*",
-        default=None,
-        help="Git revision (e.g. SHA, branch, tag)",
-        metavar="<commit>",
-    )
-    live_diff_parser.set_defaults(func=CmdLiveDiff)

dvc/command/metrics.py

@@ -12,61 +12,33 @@ DEFAULT_PRECISION = 5
 
 
 def _show_metrics(
-    metrics,
-    markdown=False,
-    all_branches=False,
-    all_tags=False,
-    all_commits=False,
-    precision=None,
+    metrics, all_branches=False, all_tags=False, all_commits=False
 ):
-    from dvc.utils.diff import format_dict, table
+    from dvc.utils.diff import format_dict
     from dvc.utils.flatten import flatten
 
     # When `metrics` contains a `None` key, it means that some files
     # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
     missing = metrics.pop(None, None)
-    with_rev = any([all_branches, all_tags, all_commits])
-    header_set = set()
-    rows = []
 
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-        return val
-
-    for _branch, val in metrics.items():
-        for _fname, metric in val.items():
-            if not isinstance(metric, dict):
-                header_set.add("")
-                continue
-            for key, _val in flatten(format_dict(metric)).items():
-                header_set.add(key)
-    header = sorted(header_set)
+    lines = []
     for branch, val in metrics.items():
-        for fname, metric in val.items():
-            row = []
-            if with_rev:
-                row.append(branch)
-            row.append(fname)
-            if not isinstance(metric, dict):
-                row.append(str(metric))
-                rows.append(row)
-                continue
-            flattened_val = flatten(format_dict(metric))
+        if all_branches or all_tags or all_commits:
+            lines.append(f"{branch}:")
 
-            for i in header:
-                row.append(_round(flattened_val.get(i)))
-            rows.append(row)
-    header.insert(0, "Path")
-    if with_rev:
-        header.insert(0, "Revision")
+        for fname, metric in val.items():
+            if not isinstance(metric, dict):
+                lines.append("\t{}: {}".format(fname, str(metric)))
+                continue
+
+            lines.append(f"\t{fname}:")
+            for key, value in flatten(format_dict(metric)).items():
+                lines.append(f"\t\t{key}: {value}")
 
     if missing:
         raise BadMetricError(missing)
-    return table(header, rows, markdown)
+
+    return "\n".join(lines)
 
 
 class CmdMetricsBase(CmdBase):
@@ -91,7 +63,6 @@ class CmdMetricsShow(CmdMetricsBase):
             else:
                 table = _show_metrics(
                     metrics,
-                    self.args.show_md,
                     self.args.all_branches,
                     self.args.all_tags,
                     self.args.all_commits,
@@ -230,12 +201,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show output in JSON format.",
     )
-    metrics_show_parser.add_argument(
-        "--show-md",
-        action="store_true",
-        default=False,
-        help="Show tabulated output in the Markdown format (GFM).",
-    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -246,15 +211,6 @@ def add_parser(subparsers, parent_parser):
             "metrics files."
         ),
     )
-    metrics_show_parser.add_argument(
-        "--precision",
-        type=int,
-        help=(
-            "Round metrics to `n` digits precision after the decimal point. "
-            f"Rounds to {DEFAULT_PRECISION} digits by default."
-        ),
-        metavar="<n>",
-    )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
     METRICS_DIFF_HELP = (
@@ -280,11 +236,8 @@ def add_parser(subparsers, parent_parser):
         "--targets",
         nargs="*",
         help=(
-            "Specific metrics file(s) to compare "
-            "(even if not found as `metrics` in `dvc.yaml`). "
-            "Using -R, directories to search metrics files in "
-            "can also be given."
-            "Shows all tracked metrics by default."
+            "Limit command scope to these metrics files. Using -R, "
+            "directories to search metrics files in can also be given."
         ),
         metavar="<paths>",
     ).complete = completion.FILE

dvc/command/move.py

@@ -25,7 +25,7 @@ def add_parser(subparsers, parent_parser):
     MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
     MOVE_DESCRIPTION = (
         "Rename or move a DVC controlled data file or a directory.\n"
-        "It renames and modifies the corresponding .dvc file to reflect the"
+        "It renames and modifies the corresponding DVC-file to reflect the"
         " changes."
     )
 

dvc/command/params.py

@@ -2,7 +2,6 @@ import argparse
 import logging
 from collections import OrderedDict
 
-from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 
@@ -31,14 +30,11 @@ def _show_diff(diff, markdown=False, no_path=False):
 
 
 class CmdParamsDiff(CmdBase):
-    UNINITIALIZED = True
-
     def run(self):
         try:
             diff = self.repo.params.diff(
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
-                targets=self.args.targets,
                 all=self.args.all,
             )
 
@@ -95,16 +91,6 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help=("New Git commit to compare (defaults to the current workspace)"),
     )
-    params_diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help=(
-            "Specific params file(s) to compare "
-            "(even if not found as `params` in `dvc.yaml`). "
-            "Shows all tracked params by default."
-        ),
-        metavar="<paths>",
-    ).complete = completion.FILE
     params_diff_parser.add_argument(
         "--all",
         action="store_true",

dvc/command/plots.py

@@ -1,29 +1,45 @@
 import argparse
 import logging
+import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.schema import PLOT_PROPS
 from dvc.utils import format_link
-from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
+PAGE_HTML = """<!DOCTYPE html>
+<html>
+<head>
+    <title>DVC Plot</title>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+</head>
+<body>
+    {divs}
+</body>
+</html>"""
+
+DIV_HTML = """<div id = "{id}"></div>
+<script type = "text/javascript">
+    var spec = {vega_json};
+    vegaEmbed('#{id}', spec);
+</script>"""
+
 
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
 
     def _props(self):
-        from dvc.schema import PLOT_PROPS
-
         # Pass only props specified by user, to not shadow ones from plot def
         props = {p: getattr(self.args, p) for p in PLOT_PROPS}
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
-        from pathlib import Path
-
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -42,24 +58,23 @@ class CmdPlots(CmdBase):
                 logger.info(plots[target])
                 return 0
 
-            rel: str = self.args.out or "plots.html"
-            path = (Path.cwd() / rel).resolve()
-            write(path, plots)
+            divs = [
+                DIV_HTML.format(id=f"plot{i}", vega_json=plot)
+                for i, plot in enumerate(plots.values())
+            ]
+            html = PAGE_HTML.format(divs="\n".join(divs))
+            path = self.args.out or "plots.html"
+
+            path = os.path.join(os.getcwd(), path)
+            with open(path, "w") as fobj:
+                fobj.write(html)
+
+            logger.info(f"file://{path}")
+
         except DvcException:
             logger.exception("")
             return 1
 
-        assert path.is_absolute()  # as_uri throws ValueError if not absolute
-        url = path.as_uri()
-        logger.info(url)
-        if self.args.open:
-            import webbrowser
-
-            opened = webbrowser.open(rel)
-            if not opened:
-                logger.error("Failed to open. Please try opening it manually.")
-                return 1
-
         return 0
 
 
@@ -143,12 +158,10 @@ def add_parser(subparsers, parent_parser):
     plots_diff_parser.add_argument(
         "--targets",
         nargs="*",
-        help=(
-            "Specific plots file(s) to visualize "
-            "(even if not found as `plots` in `dvc.yaml`). "
-            "Shows all tracked plots by default."
-        ),
-        metavar="<paths>",
+        help="Files to visualize (supports any file, "
+        "even when not found as `plots` in `dvc.yaml`). "
+        "Shows all plots by default.",
+        metavar="<path>",
     ).complete = completion.FILE
     plots_diff_parser.add_argument(
         "-e",
@@ -237,9 +250,3 @@ def _add_output_arguments(parser):
         default=False,
         help="Show output in Vega format.",
     )
-    parser.add_argument(
-        "--open",
-        action="store_true",
-        default=False,
-        help="Open plot file directly in the browser.",
-    )

dvc/command/remote.py

@@ -3,6 +3,7 @@ import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.config import ConfigError, merge
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -16,16 +17,12 @@ class CmdRemote(CmdConfig):
             self.args.name = self.args.name.lower()
 
     def _check_exists(self, conf):
-        from dvc.config import ConfigError
-
         if self.args.name not in conf["remote"]:
             raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
 
 
 class CmdRemoteAdd(CmdRemote):
     def run(self):
-        from dvc.config import ConfigError
-
         if self.args.default:
             logger.info(f"Setting '{self.args.name}' as a default remote.")
 
@@ -49,14 +46,13 @@ class CmdRemoteRemove(CmdRemote):
             self._check_exists(conf)
             del conf["remote"][self.args.name]
 
-        up_to_level = self.args.level or "repo"
         # Remove core.remote refs to this remote in any shadowing configs
         for level in reversed(self.config.LEVELS):
             with self.config.edit(level) as conf:
                 if conf["core"].get("remote") == self.args.name:
                     del conf["core"]["remote"]
 
-            if level == up_to_level:
+            if level == self.args.level:
                 break
 
         return 0
@@ -64,8 +60,6 @@ class CmdRemoteRemove(CmdRemote):
 
 class CmdRemoteModify(CmdRemote):
     def run(self):
-        from dvc.config import merge
-
         with self.config.edit(self.args.level) as conf:
             merged = self.config.load_config_to_level(self.args.level)
             merge(merged, conf)
@@ -83,10 +77,9 @@ class CmdRemoteModify(CmdRemote):
 
 class CmdRemoteDefault(CmdRemote):
     def run(self):
-        from dvc.config import ConfigError
 
         if self.args.name is None and not self.args.unset:
-            conf = self.config.read(self.args.level)
+            conf = self.config.load_one(self.args.level)
             try:
                 print(conf["core"]["remote"])
             except KeyError:
@@ -114,7 +107,7 @@ class CmdRemoteDefault(CmdRemote):
 
 class CmdRemoteList(CmdRemote):
     def run(self):
-        conf = self.config.read(self.args.level)
+        conf = self.config.load_one(self.args.level)
         for name, conf in conf["remote"].items():
             logger.info("{}\t{}".format(name, conf["url"]))
         return 0
@@ -126,8 +119,6 @@ class CmdRemoteRename(CmdRemote):
             conf["core"]["remote"] = self.args.new
 
     def run(self):
-        from dvc.config import ConfigError
-
         all_config = self.config.load_config_to_level(None)
         if self.args.new in all_config.get("remote", {}):
             raise ConfigError(
@@ -142,9 +133,8 @@ class CmdRemoteRename(CmdRemote):
             del conf["remote"][self.args.name]
             self._rename_default(conf)
 
-        up_to_level = self.args.level or "repo"
         for level in reversed(self.config.LEVELS):
-            if level == up_to_level:
+            if level == self.args.level:
                 break
             with self.config.edit(level) as level_conf:
                 self._rename_default(level_conf)

dvc/command/remove.py

@@ -14,15 +14,15 @@ class CmdRemove(CmdBase):
             try:
                 self.repo.remove(target, outs=self.args.outs)
             except DvcException:
-                logger.exception("")
+                logger.exception(f"failed to remove '{target}'")
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     REMOVE_HELP = (
-        "Remove stages from dvc.yaml and/or"
-        " stop tracking files or directories."
+        "Remove stages or .dvc files, unprotect their outputs, and"
+        " erase .gitignore entries."
     )
     remove_parser = subparsers.add_parser(
         "remove",
@@ -38,8 +38,6 @@ def add_parser(subparsers, parent_parser):
         help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "targets",
-        nargs="+",
-        help=".dvc files or stages from dvc.yaml to remove.",
+        "targets", nargs="+", help="DVC-files to remove.",
     ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -1,34 +1,56 @@
 import argparse
 import logging
+import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.metrics import _show_metrics
 from dvc.command.status import CmdDataStatus
+from dvc.dvcfile import PIPELINE_FILE
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdRepro(CmdBase):
     def run(self):
-        stages = self.repo.reproduce(**self._repro_kwargs)
-        if len(stages) == 0:
-            logger.info(CmdDataStatus.UP_TO_DATE_MSG)
-        else:
-            logger.info(
-                "Use `dvc push` to send your updates to " "remote storage."
-            )
+        saved_dir = os.path.realpath(os.curdir)
+        os.chdir(self.args.cwd)
 
-        if self.args.metrics:
-            metrics = self.repo.metrics.show()
-            logger.info(_show_metrics(metrics))
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
 
-        return 0
+        ret = 0
+        for target in self.args.targets:
+            try:
+                stages = self.repo.reproduce(target, **self._repro_kwargs)
+
+                if len(stages) == 0:
+                    logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+                else:
+                    logger.info(
+                        "Use `dvc push` to send your updates to "
+                        "remote storage."
+                    )
+
+                if self.args.metrics:
+                    metrics = self.repo.metrics.show()
+                    logger.info(_show_metrics(metrics))
+
+            except DvcException:
+                logger.exception("")
+                ret = 1
+                break
+
+        os.chdir(saved_dir)
+        return ret
 
     @property
     def _repro_kwargs(self):
         return {
-            "targets": self.args.targets,
             "single_item": self.args.single_item,
             "force": self.args.force,
             "dry": self.args.dry,
@@ -49,8 +71,8 @@ def add_arguments(repro_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="Stages to reproduce. 'dvc.yaml' by default.",
-    ).complete = completion.DVCFILES_AND_STAGE
+        help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
+    ).complete = completion.DVC_FILE
     repro_parser.add_argument(
         "-f",
         "--force",
@@ -66,6 +88,14 @@ def add_arguments(repro_parser):
         help="Reproduce only single data item without recursive dependencies "
         "check.",
     )
+    repro_parser.add_argument(
+        "-c",
+        "--cwd",
+        default=os.path.curdir,
+        help="Directory within your repo to reproduce from. Note: deprecated "
+        "by `dvc --cd <path>`.",
+        metavar="<path>",
+    )
     repro_parser.add_argument(
         "-m",
         "--metrics",
@@ -92,8 +122,8 @@ def add_arguments(repro_parser):
         "--pipeline",
         action="store_true",
         default=False,
-        help="Reproduce the whole pipeline that the specified targets "
-        "belong to.",
+        help="Reproduce the whole pipeline that the specified stage file "
+        "belongs to.",
     )
     repro_parser.add_argument(
         "-P",

dvc/command/run.py

@@ -1,8 +1,8 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.stage import parse_cmd
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -23,7 +23,7 @@ class CmdRun(CmdBase):
                 self.args.outs_persist_no_cache,
                 self.args.checkpoints,
                 self.args.params,
-                self.args.cmd,
+                self.args.command,
             ]
         ):  # pragma: no cover
             logger.error(
@@ -35,17 +35,13 @@ class CmdRun(CmdBase):
 
         try:
             self.repo.run(
-                cmd=parse_cmd(self.args.cmd),
+                cmd=self._parsed_cmd(),
                 outs=self.args.outs,
                 outs_no_cache=self.args.outs_no_cache,
                 metrics=self.args.metrics,
                 metrics_no_cache=self.args.metrics_no_cache,
                 plots=self.args.plots,
                 plots_no_cache=self.args.plots_no_cache,
-                live=self.args.live,
-                live_no_cache=self.args.live_no_cache,
-                live_no_summary=self.args.live_no_summary,
-                live_no_html=self.args.live_no_html,
                 deps=self.args.deps,
                 params=self.args.params,
                 fname=self.args.file,
@@ -69,13 +65,30 @@ class CmdRun(CmdBase):
 
         return 0
 
+    def _parsed_cmd(self):
+        """
+        We need to take into account two cases:
+
+        - ['python code.py foo bar']: Used mainly with dvc as a library
+        - ['echo', 'foo bar']: List of arguments received from the CLI
+
+        The second case would need quoting, as it was passed through:
+                dvc run echo "foo bar"
+        """
+        if len(self.args.command) < 2:
+            return " ".join(self.args.command)
+
+        return " ".join(self._quote_argument(arg) for arg in self.args.command)
+
+    def _quote_argument(self, argument):
+        if " " not in argument or '"' in argument:
+            return argument
+
+        return f'"{argument}"'
+
 
 def add_parser(subparsers, parent_parser):
-    from dvc.command.stage import _add_common_args
-
-    RUN_HELP = (
-        "Generate a dvc.yaml file from a command and execute the command."
-    )
+    RUN_HELP = "Generate a stage file from a command and execute the command."
     run_parser = subparsers.add_parser(
         "run",
         parents=[parent_parser],
@@ -83,17 +96,93 @@ def add_parser(subparsers, parent_parser):
         help=RUN_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    run_parser.add_argument(
+        "-d",
+        "--deps",
+        action="append",
+        default=[],
+        help="Declare dependencies for reproducible cmd.",
+        metavar="<path>",
+    ).complete = completion.FILE
     run_parser.add_argument(
         "-n", "--name", help="Stage name.",
     )
+    run_parser.add_argument(
+        "-o",
+        "--outs",
+        action="append",
+        default=[],
+        help="Declare output file or directory.",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    run_parser.add_argument(
+        "-O",
+        "--outs-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory "
+        "(do not put into DVC cache).",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    run_parser.add_argument(
+        "-p",
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter to use as additional dependency.",
+        metavar="[<filename>:]<params_list>",
+    ).complete = completion.FILE
+    run_parser.add_argument(
+        "-m",
+        "--metrics",
+        action="append",
+        default=[],
+        help="Declare output metrics file.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-M",
+        "--metrics-no-cache",
+        action="append",
+        default=[],
+        help="Declare output metrics file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "--plots",
+        action="append",
+        default=[],
+        help="Declare output plot file.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "--plots-no-cache",
+        action="append",
+        default=[],
+        help="Declare output plot file (do not put into DVC cache).",
+        metavar="<path>",
+    )
     run_parser.add_argument(
         "--file", metavar="<filename>", help=argparse.SUPPRESS,
     )
+    run_parser.add_argument(
+        "-w",
+        "--wdir",
+        help="Directory within your repo to run your command in.",
+        metavar="<path>",
+    )
     run_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create dvc.yaml without actually running it.",
+        help="Only create stage file without actually running it.",
+    )
+    run_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite existing stage",
     )
     run_parser.add_argument(
         "--no-run-cache",
@@ -110,11 +199,58 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
+    run_parser.add_argument(
+        "--outs-persist",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro.",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "--outs-persist-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro (do not put into DVC cache).",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "-c",
+        "--checkpoints",
+        action="append",
+        default=[],
+        help=argparse.SUPPRESS,
+        metavar="<filename>",
+    ).complete = completion.FILE
+    run_parser.add_argument(
+        "--always-changed",
+        action="store_true",
+        default=False,
+        help="Always consider this DVC-file as changed.",
+    )
     run_parser.add_argument(
         "--single-stage",
         action="store_true",
         default=False,
         help=argparse.SUPPRESS,
     )
-    _add_common_args(run_parser)
+    run_parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow outputs that are outside of the DVC repository.",
+    )
+    run_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the stage (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    run_parser.add_argument(
+        "command", nargs=argparse.REMAINDER, help="Command to execute."
+    )
     run_parser.set_defaults(func=CmdRun)

dvc/command/stage.py

@@ -1,365 +0,0 @@
-import argparse
-import logging
-from itertools import chain, filterfalse
-from typing import TYPE_CHECKING, Dict, Iterable, List
-
-from dvc.command import completion
-from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.utils.cli_parse import parse_params
-from dvc.utils.humanize import truncate_text
-
-if TYPE_CHECKING:
-    from dvc.output.base import BaseOutput
-    from dvc.stage import Stage
-
-logger = logging.getLogger(__name__)
-
-MAX_TEXT_LENGTH = 80
-
-
-def generate_description(stage: "Stage") -> str:
-    def part_desc(outs: Iterable["BaseOutput"]) -> str:
-        return ", ".join(out.def_path for out in outs)
-
-    if not stage.deps and not stage.outs:
-        return "No outputs or dependencies"
-
-    if not stage.outs and stage.deps:
-        return "Depends on " + part_desc(stage.deps)
-
-    def is_plot_or_metric(out: "BaseOutput"):
-        return bool(out.plot) or bool(out.metric)
-
-    desc: List[str] = []
-
-    outs = list(filterfalse(is_plot_or_metric, stage.outs))
-    if outs:
-        desc.append("Outputs " + part_desc(outs))
-
-    plots_and_metrics = list(filter(is_plot_or_metric, stage.outs))
-    if plots_and_metrics:
-        desc.append("Reports " + part_desc(plots_and_metrics))
-
-    return "; ".join(desc)
-
-
-def prepare_description(
-    stage: "Stage", max_length: int = MAX_TEXT_LENGTH
-) -> str:
-    desc = stage.short_description() or generate_description(stage)
-    return truncate_text(desc, max_length)
-
-
-def prepare_stages_data(
-    stages: Iterable["Stage"],
-    description: bool = True,
-    max_length: int = MAX_TEXT_LENGTH,
-) -> Dict[str, str]:
-    return {
-        stage.addressing: prepare_description(stage, max_length=max_length)
-        if description
-        else ""
-        for stage in stages
-    }
-
-
-class CmdStageList(CmdBase):
-    def _get_stages(self) -> Iterable["Stage"]:
-        if self.args.all:
-            stages: List["Stage"] = self.repo.stages  # type: ignore
-            logger.trace(  # type: ignore[attr-defined]
-                "%d no. of stages found", len(stages)
-            )
-            return stages
-
-        # removing duplicates while maintaining order
-        collected = chain.from_iterable(
-            self.repo.stage.collect(
-                target=target,
-                recursive=self.args.recursive,
-                accept_group=True,
-            )
-            for target in self.args.targets
-        )
-        return dict.fromkeys(collected).keys()
-
-    def run(self):
-        from dvc.utils.diff import table
-
-        def log_error(relpath: str, exc: Exception):
-            if self.args.fail:
-                raise exc
-            logger.debug("Stages from %s failed to load", relpath)
-
-        # silence stage collection error by default
-        self.repo.stage_collection_error_handler = log_error
-
-        stages = self._get_stages()
-        names_only = self.args.names_only
-
-        data = prepare_stages_data(stages, description=not names_only)
-        if data:
-            print(table(header=(), rows=data.items()))
-
-        return 0
-
-
-def parse_cmd(commands: List[str]) -> str:
-    """
-    We need to take into account two cases:
-
-    - ['python code.py foo bar']: Used mainly with dvc as a library
-    - ['echo', 'foo bar']: List of arguments received from the CLI
-
-    The second case would need quoting, as it was passed through:
-            dvc run echo "foo bar"
-    """
-
-    def quote_argument(arg: str):
-        should_quote = " " in arg and '"' not in arg
-        return f'"{arg}"' if should_quote else arg
-
-    if len(commands) < 2:
-        return " ".join(commands)
-    return " ".join(map(quote_argument, commands))
-
-
-class CmdStageAdd(CmdBase):
-    def run(self):
-        kwargs = vars(self.args)
-        kwargs.update(
-            {
-                "cmd": parse_cmd(kwargs.pop("cmd")),
-                "params": parse_params(self.args.params),
-            }
-        )
-        self.repo.stage.add(**kwargs)
-        return 0
-
-
-def _add_common_args(parser):
-    parser.add_argument(
-        "-d",
-        "--deps",
-        action="append",
-        default=[],
-        help="Declare dependencies for reproducible cmd.",
-        metavar="<path>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-o",
-        "--outs",
-        action="append",
-        default=[],
-        help="Declare output file or directory.",
-        metavar="<filename>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-O",
-        "--outs-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory "
-        "(do not put into DVC cache).",
-        metavar="<filename>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-p",
-        "--params",
-        action="append",
-        default=[],
-        help="Declare parameter to use as additional dependency.",
-        metavar="[<filename>:]<params_list>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-m",
-        "--metrics",
-        action="append",
-        default=[],
-        help="Declare output metrics file.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "-M",
-        "--metrics-no-cache",
-        action="append",
-        default=[],
-        help="Declare output metrics file (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--plots",
-        action="append",
-        default=[],
-        help="Declare output plot file.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--plots-no-cache",
-        action="append",
-        default=[],
-        help="Declare output plot file (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--live", help="Declare output as dvclive.", metavar="<path>",
-    )
-    parser.add_argument(
-        "--live-no-cache",
-        help="Declare output as dvclive (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--live-no-summary",
-        action="store_true",
-        default=False,
-        help="Signal dvclive logger to not dump latest metrics file.",
-    )
-    parser.add_argument(
-        "--live-no-html",
-        action="store_true",
-        default=False,
-        help="Signal dvclive logger to not produce training report.",
-    )
-    parser.add_argument(
-        "-w",
-        "--wdir",
-        help="Directory within your repo to run your command in.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite existing stage",
-    )
-    parser.add_argument(
-        "--outs-persist",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro.",
-        metavar="<filename>",
-    )
-    parser.add_argument(
-        "--outs-persist-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro (do not put into DVC cache).",
-        metavar="<filename>",
-    )
-    parser.add_argument(
-        "-c",
-        "--checkpoints",
-        action="append",
-        default=[],
-        help="Declare checkpoint output file or directory for 'dvc exp run'. "
-        "Not compatible with 'dvc repro'.",
-        metavar="<filename>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "--always-changed",
-        action="store_true",
-        default=False,
-        help="Always consider this DVC-file as changed.",
-    )
-    parser.add_argument(
-        "--external",
-        action="store_true",
-        default=False,
-        help="Allow outputs that are outside of the DVC repository.",
-    )
-    parser.add_argument(
-        "--desc",
-        type=str,
-        metavar="<text>",
-        help=(
-            "User description of the stage (optional). "
-            "This doesn't affect any DVC operations."
-        ),
-    )
-    parser.add_argument(
-        "cmd",
-        nargs=argparse.REMAINDER,
-        help="Command to execute.",
-        metavar="command",
-    )
-
-
-def add_parser(subparsers, parent_parser):
-    STAGES_HELP = "Commands to list and create stages."
-
-    stage_parser = subparsers.add_parser(
-        "stage",
-        parents=[parent_parser],
-        description=append_doc_link(STAGES_HELP, "stage"),
-        help=STAGES_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    stage_subparsers = stage_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc stage CMD --help` to display command-specific help.",
-    )
-
-    fix_subparsers(stage_subparsers)
-
-    STAGE_ADD_HELP = "Create stage"
-    stage_add_parser = stage_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(STAGE_ADD_HELP, "stage/add"),
-        help=STAGE_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    stage_add_parser.add_argument(
-        "-n", "--name", help="Name of the stage to add", required=True
-    )
-    _add_common_args(stage_add_parser)
-    stage_add_parser.set_defaults(func=CmdStageAdd)
-
-    STAGE_LIST_HELP = "List stages."
-    stage_list_parser = stage_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(STAGE_LIST_HELP, "stage/list"),
-        help=STAGE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    stage_list_parser.add_argument(
-        "targets",
-        nargs="*",
-        default=["dvc.yaml"],
-        help=(
-            "Show stages from a dvc.yaml/.dvc file or a directory. "
-            "'dvc.yaml' by default"
-        ),
-    )
-    stage_list_parser.add_argument(
-        "--all",
-        action="store_true",
-        default=False,
-        help="List all of the stages in the repo.",
-    )
-    stage_list_parser.add_argument(
-        "--fail",
-        action="store_true",
-        default=False,
-        help="Fail immediately, do not suppress any syntax errors.",
-    )
-    stage_list_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="List all stages inside the specified directory.",
-    )
-    stage_list_parser.add_argument(
-        "--names-only",
-        action="store_true",
-        default=False,
-        help="List only stage names.",
-    )
-    stage_list_parser.set_defaults(func=CmdStageList)

dvc/command/status.py

@@ -44,8 +44,6 @@ class CmdDataStatus(CmdDataBase):
                 self._show(value, indent + 1)
 
     def run(self):
-        from dvc.ui import ui
-
         indent = 1 if self.args.cloud else 0
         try:
             st = self.repo.status(
@@ -70,14 +68,14 @@ class CmdDataStatus(CmdDataBase):
             elif st:
                 self._show(st, indent)
             elif not self.repo.stages:
-                ui.write(self.EMPTY_PROJECT_MSG)
+                logger.info(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
                     "remote"
                 )
-                ui.write(self.IN_SYNC_MSG.format(remote=remote))
+                logger.info(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                ui.write(self.UP_TO_DATE_MSG)
+                logger.info(self.UP_TO_DATE_MSG)
 
         except DvcException:
             logger.exception("")

dvc/command/update.py

@@ -16,9 +16,6 @@ class CmdUpdate(CmdBase):
                 targets=self.args.targets,
                 rev=self.args.rev,
                 recursive=self.args.recursive,
-                to_remote=self.args.to_remote,
-                remote=self.args.remote,
-                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception("failed update data")
@@ -36,7 +33,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help=".dvc files to update.",
+        "targets", nargs="+", help="DVC-files to update.",
     ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",
@@ -51,27 +48,4 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Update all stages in the specified directory.",
     )
-    update_parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Update data directly on the remote",
-    )
-    update_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to perform updates to",
-        metavar="<name>",
-    )
-    update_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-            "For SSH remotes, the default is 4. "
-        ),
-        metavar="<number>",
-    )
     update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -2,14 +2,13 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.info import get_dvc_info
 
 logger = logging.getLogger(__name__)
 
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
-        from dvc.info import get_dvc_info
-
         dvc_info = get_dvc_info()
         logger.info(dvc_info)
         return 0

dvc/config.py

@@ -4,11 +4,25 @@ import os
 import re
 from contextlib import contextmanager
 from functools import partial
+from urllib.parse import urlparse
 
-from funcy import cached_property, compact, memoize, re_find
+import configobj
+from funcy import cached_property, compact, re_find, walk_values
+from voluptuous import (
+    ALLOW_EXTRA,
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
 
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.path_info import PathInfo
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -24,19 +38,205 @@ class NoRemoteError(ConfigError):
     pass
 
 
-@memoize
-def get_compiled_schema():
-    from voluptuous import Schema
+def supported_cache_type(types):
+    """Checks if link type config option consists only of valid values.
 
-    from .config_schema import SCHEMA
+    Args:
+        types (list/string): type(s) of links that dvc should try out.
+    """
+    if types is None:
+        return None
+    if isinstance(types, str):
+        types = [typ.strip() for typ in types.split(",")]
 
-    return Schema(SCHEMA)
+    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
+    if unsupported:
+        raise Invalid(
+            "Unsupported cache type(s): {}".format(", ".join(unsupported))
+        )
+
+    return types
 
 
-def to_bool(value):
-    from .config_schema import Bool
+# Checks that value is either true or false and converts it to bool
+to_bool = Bool = All(
+    Lower,
+    Any("true", "false"),
+    lambda v: v == "true",
+    msg="expected true or false",
+)
 
-    return Bool(value)
+
+def Choices(*choices):
+    """Checks that value belongs to the specified set of values
+
+    Args:
+        *choices: pass allowed values as arguments, or pass a list or
+            tuple as a single argument
+    """
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
+
+
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    "jobs": All(Coerce(int), Range(1)),
+    Optional("no_traverse"): Bool,  # obsoleted
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "ssl_verify": Bool,
+    "method": str,
+}
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+}
+
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+        Optional("autostage", default=False): Bool,
+        Optional("experiments", default=False): Bool,
+        Optional("check_update", default=True): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "webhdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    "access_key_id": str,
+                    "secret_access_key": str,
+                    Optional("listobjects", default=False): Bool,  # obsoleted
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "sse_kms_key_id": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    "allow_agent": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "webhdfs": {
+                    "hdfscli_config": str,
+                    "webhdfs_token": str,
+                    "user": str,
+                    "webhdfs_alias": str,
+                    **REMOTE_COMMON,
+                },
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_email": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_p12_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+    # section for experimental features
+    "feature": {Optional("parametrization", default=False): Bool},
+}
+COMPILED_SCHEMA = Schema(SCHEMA)
 
 
 class Config(dict):
@@ -63,9 +263,9 @@ class Config(dict):
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True, fs=None, config=None,
+        self, dvc_dir=None, validate=True, tree=None,
     ):  # pylint: disable=super-init-not-called
-        from dvc.fs.local import LocalFileSystem
+        from dvc.tree.local import LocalTree
 
         self.dvc_dir = dvc_dir
 
@@ -79,10 +279,10 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.wfs = LocalFileSystem(None, {"url": self.dvc_dir})
-        self.fs = fs or self.wfs
+        self.wtree = LocalTree(None, {"url": self.dvc_dir})
+        self.tree = tree or self.wtree
 
-        self.load(validate=validate, config=config)
+        self.load(validate=validate)
 
     @classmethod
     def get_dir(cls, level):
@@ -122,7 +322,7 @@ class Config(dict):
         open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def load(self, validate=True, config=None):
+    def load(self, validate=True):
         """Loads config from all the config files.
 
         Raises:
@@ -130,9 +330,6 @@ class Config(dict):
         """
         conf = self.load_config_to_level()
 
-        if config is not None:
-            merge(conf, config)
-
         if validate:
             conf = self.validate(conf)
 
@@ -143,36 +340,32 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
-    def _get_fs(self, level):
-        # NOTE: this might be a Gitfs, which doesn't see things outside of
+    def _get_tree(self, level):
+        # NOTE: this might be a GitTree, which doesn't see things outside of
         # the repo.
-        return self.fs if level == "repo" else self.wfs
+        return self.tree if level == "repo" else self.wtree
 
     def _load_config(self, level):
-        from configobj import ConfigObj
-
         filename = self.files[level]
-        fs = self._get_fs(level)
+        tree = self._get_tree(level)
 
-        if fs.exists(filename, use_dvcignore=False):
-            with fs.open(filename) as fobj:
-                conf_obj = ConfigObj(fobj)
+        if tree.exists(filename, use_dvcignore=False):
+            with tree.open(filename) as fobj:
+                conf_obj = configobj.ConfigObj(fobj)
         else:
-            conf_obj = ConfigObj()
+            conf_obj = configobj.ConfigObj()
         return _parse_remotes(_lower_keys(conf_obj.dict()))
 
     def _save_config(self, level, conf_dict):
-        from configobj import ConfigObj
-
         filename = self.files[level]
-        fs = self._get_fs(level)
+        tree = self._get_tree(level)
 
         logger.debug(f"Writing '{filename}'.")
 
-        fs.makedirs(os.path.dirname(filename))
+        tree.makedirs(os.path.dirname(filename))
 
-        config = ConfigObj(_pack_remotes(conf_dict))
-        with fs.open(filename, "wb") as fobj:
+        config = configobj.ConfigObj(_pack_remotes(conf_dict))
+        with tree.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
 
@@ -181,7 +374,7 @@ class Config(dict):
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
-        for key in get_compiled_schema().schema:
+        for key in COMPILED_SCHEMA.schema:
             conf.setdefault(key, {})
 
         return conf
@@ -191,8 +384,6 @@ class Config(dict):
         abs_conf_dir = os.path.abspath(os.path.dirname(filename))
 
         def resolve(path):
-            from .config_schema import RelPath
-
             if os.path.isabs(path) or re.match(r"\w+://", path):
                 return path
 
@@ -207,10 +398,6 @@ class Config(dict):
 
     @staticmethod
     def _to_relpath(conf_dir, path):
-        from dvc.utils import relpath
-
-        from .config_schema import RelPath
-
         if re.match(r"\w+://", path):
             return path
 
@@ -228,15 +415,13 @@ class Config(dict):
 
     @staticmethod
     def _map_dirs(conf, func):
-        from voluptuous import ALLOW_EXTRA, Schema
-
         dirs_schema = {
             "cache": {"dir": func},
             "remote": {
                 str: {
                     "url": func,
                     "gdrive_user_credentials_file": func,
-                    "gdrive_service_account_json_file_path": func,
+                    "gdrive_service_account_p12_file_path": func,
                     "credentialpath": func,
                     "keyfile": func,
                     "cert_path": func,
@@ -255,16 +440,8 @@ class Config(dict):
                 merge(merged_conf, self.load_one(merge_level))
         return merged_conf
 
-    def read(self, level=None):
-        # NOTE: we read from a merged config by default, same as git config
-        if level is None:
-            return self.load_config_to_level()
-        return self.load_one(level)
-
     @contextmanager
-    def edit(self, level=None, validate=True):
-        # NOTE: we write to repo config by default, same as git config
-        level = level or "repo"
+    def edit(self, level="repo"):
         if level in {"repo", "local"} and self.dvc_dir is None:
             raise ConfigError("Not inside a DVC repo")
 
@@ -275,19 +452,15 @@ class Config(dict):
 
         merged_conf = self.load_config_to_level(level)
         merge(merged_conf, conf)
-
-        if validate:
-            self.validate(merged_conf)
+        self.validate(merged_conf)
 
         self._save_config(level, conf)
-        self.load(validate=validate)
+        self.load()
 
     @staticmethod
     def validate(data):
-        from voluptuous import Invalid
-
         try:
-            return get_compiled_schema()(data)
+            return COMPILED_SCHEMA(data)
         except Invalid as exc:
             raise ConfigError(str(exc)) from None
 

dvc/config_schema.py

@@ -1,225 +0,0 @@
-import os
-from urllib.parse import urlparse
-
-from funcy import walk_values
-from voluptuous import (
-    All,
-    Any,
-    Coerce,
-    Invalid,
-    Lower,
-    Optional,
-    Range,
-    Schema,
-)
-
-Bool = All(
-    Lower,
-    Any("true", "false"),
-    lambda v: v == "true",
-    msg="expected true or false",
-)
-
-
-def supported_cache_type(types):
-    """Checks if link type config option consists only of valid values.
-
-    Args:
-        types (list/string): type(s) of links that dvc should try out.
-    """
-    if types is None:
-        return None
-    if isinstance(types, str):
-        types = [typ.strip() for typ in types.split(",")]
-
-    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
-    if unsupported:
-        raise Invalid(
-            "Unsupported cache type(s): {}".format(", ".join(unsupported))
-        )
-
-    return types
-
-
-def Choices(*choices):
-    """Checks that value belongs to the specified set of values
-
-    Args:
-        *choices: pass allowed values as arguments, or pass a list or
-            tuple as a single argument
-    """
-    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
-
-
-def ByUrl(mapping):
-    schemas = walk_values(Schema, mapping)
-
-    def validate(data):
-        if "url" not in data:
-            raise Invalid("expected 'url'")
-
-        parsed = urlparse(data["url"])
-        # Windows absolute paths should really have scheme == "" (local)
-        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
-            return schemas[""](data)
-        if parsed.scheme not in schemas:
-            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
-
-        return schemas[parsed.scheme](data)
-
-    return validate
-
-
-class RelPath(str):
-    pass
-
-
-REMOTE_COMMON = {
-    "url": str,
-    "checksum_jobs": All(Coerce(int), Range(1)),
-    "jobs": All(Coerce(int), Range(1)),
-    Optional("no_traverse"): Bool,  # obsoleted
-    "verify": Bool,
-}
-LOCAL_COMMON = {
-    "type": supported_cache_type,
-    Optional("protected", default=False): Bool,  # obsoleted
-    "shared": All(Lower, Choices("group")),
-    Optional("slow_link_warning", default=True): Bool,
-}
-HTTP_COMMON = {
-    "auth": All(Lower, Choices("basic", "digest", "custom")),
-    "custom_auth_header": str,
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "ssl_verify": Bool,
-    "method": str,
-}
-WEBDAV_COMMON = {
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "token": str,
-    "cert_path": str,
-    "key_path": str,
-    "timeout": Coerce(int),
-}
-
-SCHEMA = {
-    "core": {
-        "remote": Lower,
-        "checksum_jobs": All(Coerce(int), Range(1)),
-        Optional("interactive", default=False): Bool,
-        Optional("analytics", default=True): Bool,
-        Optional("hardlink_lock", default=False): Bool,
-        Optional("no_scm", default=False): Bool,
-        Optional("autostage", default=False): Bool,
-        Optional("experiments"): Bool,  # obsoleted
-        Optional("check_update", default=True): Bool,
-    },
-    "cache": {
-        "local": str,
-        "s3": str,
-        "gs": str,
-        "hdfs": str,
-        "webhdfs": str,
-        "ssh": str,
-        "azure": str,
-        # This is for default local cache
-        "dir": str,
-        **LOCAL_COMMON,
-    },
-    "remote": {
-        str: ByUrl(
-            {
-                "": {**LOCAL_COMMON, **REMOTE_COMMON},
-                "s3": {
-                    "region": str,
-                    "profile": str,
-                    "credentialpath": str,
-                    "configpath": str,
-                    "endpointurl": str,
-                    "access_key_id": str,
-                    "secret_access_key": str,
-                    "session_token": str,
-                    Optional("listobjects", default=False): Bool,  # obsoleted
-                    Optional("use_ssl", default=True): Bool,
-                    "sse": str,
-                    "sse_kms_key_id": str,
-                    "acl": str,
-                    "grant_read": str,
-                    "grant_read_acp": str,
-                    "grant_write_acp": str,
-                    "grant_full_control": str,
-                    **REMOTE_COMMON,
-                },
-                "gs": {
-                    "projectname": str,
-                    "credentialpath": str,
-                    **REMOTE_COMMON,
-                },
-                "ssh": {
-                    "type": supported_cache_type,
-                    "port": Coerce(int),
-                    "user": str,
-                    "password": str,
-                    "ask_password": Bool,
-                    "keyfile": str,
-                    "timeout": Coerce(int),
-                    "gss_auth": Bool,
-                    "allow_agent": Bool,
-                    **REMOTE_COMMON,
-                },
-                "hdfs": {"user": str, **REMOTE_COMMON},
-                "webhdfs": {
-                    "hdfscli_config": str,
-                    "webhdfs_token": str,
-                    "user": str,
-                    "webhdfs_alias": str,
-                    **REMOTE_COMMON,
-                },
-                "azure": {
-                    "connection_string": str,
-                    "sas_token": str,
-                    "account_name": str,
-                    "account_key": str,
-                    "tenant_id": str,
-                    "client_id": str,
-                    "client_secret": str,
-                    **REMOTE_COMMON,
-                },
-                "oss": {
-                    "oss_key_id": str,
-                    "oss_key_secret": str,
-                    "oss_endpoint": str,
-                    **REMOTE_COMMON,
-                },
-                "gdrive": {
-                    "gdrive_use_service_account": Bool,
-                    "gdrive_client_id": str,
-                    "gdrive_client_secret": str,
-                    "gdrive_user_credentials_file": str,
-                    "gdrive_service_account_user_email": str,
-                    "gdrive_service_account_json_file_path": str,
-                    Optional("gdrive_trash_only", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "http": {**HTTP_COMMON, **REMOTE_COMMON},
-                "https": {**HTTP_COMMON, **REMOTE_COMMON},
-                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "remote": {str: object},  # Any of the above options are valid
-            }
-        )
-    },
-    "state": {
-        "row_limit": All(Coerce(int), Range(1)),  # obsoleted
-        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
-    },
-    # section for experimental features
-    "feature": {
-        # enabled by default. It's of no use, kept for backward compatibility.
-        Optional("parametrization", default=True): Bool
-    },
-}

dvc/data_cloud.py

@@ -2,6 +2,9 @@
 
 import logging
 
+from dvc.config import NoRemoteError
+from dvc.remote import get_remote
+
 logger = logging.getLogger(__name__)
 
 
@@ -20,8 +23,6 @@ class DataCloud:
         self.repo = repo
 
     def get_remote(self, name=None, command="<command>"):
-        from dvc.config import NoRemoteError
-
         if not name:
             name = self.repo.config["core"].get("remote")
 
@@ -44,8 +45,6 @@ class DataCloud:
         raise NoRemoteError(error_msg)
 
     def _init_remote(self, name):
-        from dvc.remote import get_remote
-
         return get_remote(self.repo, name=name)
 
     def push(
@@ -64,7 +63,7 @@ class DataCloud:
         remote = self.get_remote(remote, "push")
 
         return remote.push(
-            self.repo.odb.local,
+            self.repo.cache.local,
             cache,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -86,30 +85,12 @@ class DataCloud:
         remote = self.get_remote(remote, "pull")
 
         return remote.pull(
-            self.repo.odb.local,
+            self.repo.cache.local,
             cache,
             jobs=jobs,
             show_checksums=show_checksums,
         )
 
-    def transfer(self, source, jobs=None, remote=None, command=None):
-        """Transfer data items in a cloud-agnostic way.
-
-        Args:
-            source (str): url for the source location.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to compare
-                cache to. By default remote from core.remote config option
-                is used.
-            command (str): the command which is benefitting from this function
-                (to be used for reporting better error messages).
-        """
-        from dvc.fs import get_cloud_fs
-
-        from_fs = get_cloud_fs(self.repo, url=source)
-        remote = self.get_remote(remote, command)
-        return remote.transfer(from_fs, from_fs.path_info, jobs=jobs)
-
     def status(
         self,
         cache,
@@ -133,7 +114,7 @@ class DataCloud:
         """
         remote = self.get_remote(remote, "status")
         return remote.status(
-            self.repo.odb.local,
+            self.repo.cache.local,
             cache,
             jobs=jobs,
             show_checksums=show_checksums,
@@ -142,4 +123,4 @@ class DataCloud:
 
     def get_url_for(self, remote, checksum):
         remote = self.get_remote(remote)
-        return str(remote.odb.hash_to_path_info(checksum))
+        return str(remote.tree.hash_to_path_info(checksum))

dvc/dependency/__init__.py

@@ -17,9 +17,23 @@ from dvc.dependency.webhdfs import WebHDFSDependency
 from dvc.output.base import BaseOutput
 from dvc.scheme import Schemes
 
-from ..fs import get_cloud_fs
+from ..tree import get_cloud_tree
 from .repo import RepoDependency
 
+DEPS = [
+    AzureDependency,
+    GSDependency,
+    HDFSDependency,
+    HTTPDependency,
+    HTTPSDependency,
+    S3Dependency,
+    SSHDependency,
+    WebDAVDependency,
+    WebDAVSDependency,
+    WebHDFSDependency,
+    # NOTE: LocalDependency is the default choice
+]
+
 DEP_MAP = {
     Schemes.LOCAL: LocalDependency,
     Schemes.SSH: SSHDependency,
@@ -50,8 +64,8 @@ SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 def _get(stage, p, info):
     parsed = urlparse(p) if p else None
     if parsed and parsed.scheme == "remote":
-        fs = get_cloud_fs(stage.repo, name=parsed.netloc)
-        return DEP_MAP[fs.scheme](stage, p, info, fs=fs)
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
 
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)
@@ -61,8 +75,10 @@ def _get(stage, p, info):
         params = info.pop(ParamsDependency.PARAM_PARAMS)
         return ParamsDependency(stage, p, params)
 
-    dep_cls = DEP_MAP.get(parsed.scheme, LocalDependency)
-    return dep_cls(stage, p, info)
+    for d in DEPS:
+        if d.supported(p):
+            return d(stage, p, info)
+    return LocalDependency(stage, p, info)
 
 
 def loadd_from(stage, d_list):

dvc/dependency/azure.py

@@ -1,8 +1,8 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
 
-from ..fs.azure import AzureFileSystem
+from ..tree.azure import AzureTree
 
 
 class AzureDependency(BaseDependency, BaseOutput):
-    FS_CLS = AzureFileSystem
+    TREE_CLS = AzureTree

dvc/dependency/base.py

@@ -17,7 +17,7 @@ class DependencyIsNotFileOrDirError(DvcException):
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__(f"DVC file '{path}' cannot be a dependency.")
+        super().__init__(f"Stage file '{path}' cannot be a dependency.")
 
 
 class BaseDependency:

dvc/dependency/gs.py

@@ -1,8 +1,6 @@
 from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..fs.gs import GSFileSystem
+from dvc.output.gs import GSOutput
 
 
-class GSDependency(BaseDependency, BaseOutput):
-    FS_CLS = GSFileSystem
+class GSDependency(BaseDependency, GSOutput):
+    pass

dvc/dependency/http.py

@@ -1,8 +1,8 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
 
-from ..fs.http import HTTPFileSystem
+from ..tree.http import HTTPTree
 
 
 class HTTPDependency(BaseDependency, BaseOutput):
-    FS_CLS = HTTPFileSystem
+    TREE_CLS = HTTPTree

dvc/dependency/https.py

@@ -1,6 +1,6 @@
-from ..fs.https import HTTPSFileSystem
+from ..tree.https import HTTPSTree
 from .http import HTTPDependency
 
 
 class HTTPSDependency(HTTPDependency):
-    FS_CLS = HTTPSFileSystem
+    TREE_CLS = HTTPSTree

dvc/dependency/param.py

@@ -91,7 +91,7 @@ class ParamsDependency(LocalDependency):
         suffix = self.path_info.suffix.lower()
         loader = LOADERS[suffix]
         try:
-            config = loader(self.path_info, fs=self.repo.fs)
+            config = loader(self.path_info, tree=self.repo.tree)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"

dvc/dependency/repo.py

@@ -1,7 +1,5 @@
 from voluptuous import Required
 
-from dvc.path_info import PathInfo
-
 from .local import LocalDependency
 
 
@@ -23,7 +21,7 @@ class RepoDependency(LocalDependency):
         self.def_repo = def_repo
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, fs, path):
+    def _parse_path(self, tree, path):
         return None
 
     @property
@@ -47,17 +45,10 @@ class RepoDependency(LocalDependency):
         return external_repo(d["url"], rev=rev, **kwargs)
 
     def _get_hash(self, locked=True):
-        from dvc.objects.stage import stage
-
-        with self._make_repo(locked=locked) as repo:
-            path_info = PathInfo(repo.root_dir) / self.def_path
-            return stage(
-                self.repo.odb.local,
-                path_info,
-                repo.repo_fs,
-                self.repo.odb.local.fs.PARAM_CHECKSUM,
-                follow_subrepos=False,
-            ).hash_info
+        # we want stream but not fetch, so DVC out directories are
+        # walked, but dir contents is not fetched
+        with self._make_repo(locked=locked, fetch=False, stream=True) as repo:
+            return repo.get_checksum(self.def_path)
 
     def workspace_status(self):
         current = self._get_hash(locked=True)
@@ -77,34 +68,16 @@ class RepoDependency(LocalDependency):
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def download(self, to, jobs=None):
-        from dvc.checkout import checkout
-        from dvc.config import NoRemoteError
-        from dvc.exceptions import NoOutputOrStageError
-        from dvc.objects import save
-        from dvc.objects.stage import stage
+    def download(self, to):
+        cache = self.repo.cache.local
 
-        odb = self.repo.odb.local
-
-        with self._make_repo(cache_dir=odb.cache_dir) as repo:
+        with self._make_repo(cache_dir=cache.cache_dir) as repo:
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
                 self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
-            path_info = PathInfo(repo.root_dir) / self.def_path
-            try:
-                repo.fetch([path_info.fspath], jobs=jobs, recursive=True)
-            except (NoOutputOrStageError, NoRemoteError):
-                pass
-            obj = stage(
-                odb,
-                path_info,
-                repo.repo_fs,
-                odb.fs.PARAM_CHECKSUM,
-                jobs=jobs,
-                follow_subrepos=False,
-            )
-            save(odb, obj, jobs=jobs)
 
-        checkout(to.path_info, to.fs, obj, odb)
+            _, _, cache_infos = repo.fetch_external([self.def_path])
+
+        cache.checkout(to.path_info, cache_infos[0])
 
     def update(self, rev=None):
         if rev:

dvc/dependency/webdav.py

@@ -1,8 +1,8 @@
 from dvc.dependency.base import BaseDependency
 from dvc.output.base import BaseOutput
 
-from ..fs.webdav import WebDAVFileSystem
+from ..tree.webdav import WebDAVTree
 
 
 class WebDAVDependency(BaseDependency, BaseOutput):
-    FS_CLS = WebDAVFileSystem
+    TREE_CLS = WebDAVTree

dvc/dependency/webdavs.py

@@ -1,6 +1,6 @@
-from ..fs.webdavs import WebDAVSFileSystem
+from ..tree.webdavs import WebDAVSTree
 from .webdav import WebDAVDependency
 
 
 class WebDAVSDependency(WebDAVDependency):
-    FS_CLS = WebDAVSFileSystem
+    TREE_CLS = WebDAVSTree

dvc/dir_info.py

@@ -0,0 +1,102 @@
+import posixpath
+from operator import itemgetter
+
+from pygtrie import Trie
+
+from .hash_info import HashInfo
+
+
+def _diff(ancestor, other, allow_removed=False):
+    from dictdiffer import diff
+
+    from dvc.exceptions import MergeError
+
+    allowed = ["add"]
+    if allow_removed:
+        allowed.append("remove")
+
+    result = list(diff(ancestor, other))
+    for typ, _, _ in result:
+        if typ not in allowed:
+            raise MergeError(
+                "unable to auto-merge directories with diff that contains "
+                f"'{typ}'ed files"
+            )
+    return result
+
+
+def _merge(ancestor, our, their):
+    import copy
+
+    from dictdiffer import patch
+
+    our_diff = _diff(ancestor, our)
+    if not our_diff:
+        return copy.deepcopy(their)
+
+    their_diff = _diff(ancestor, their)
+    if not their_diff:
+        return copy.deepcopy(our)
+
+    # make sure there are no conflicting files
+    _diff(our, their, allow_removed=True)
+
+    return patch(our_diff + their_diff, ancestor)
+
+
+class DirInfo:
+    PARAM_RELPATH = "relpath"
+
+    def __init__(self):
+        self.trie = Trie()
+
+    @property
+    def size(self):
+        try:
+            return sum(
+                hash_info.size
+                for _, hash_info in self.trie.iteritems()  # noqa: B301
+            )
+        except TypeError:
+            return None
+
+    @property
+    def nfiles(self):
+        return len(self.trie)
+
+    def items(self, path_info=None):
+        for key, hash_info in self.trie.iteritems():  # noqa: B301
+            path = posixpath.sep.join(key)
+            if path_info is not None:
+                path = path_info / path
+            yield path, hash_info
+
+    @classmethod
+    def from_list(cls, lst):
+        ret = DirInfo()
+        for _entry in lst:
+            entry = _entry.copy()
+            relpath = entry.pop(cls.PARAM_RELPATH)
+            parts = tuple(relpath.split(posixpath.sep))
+            ret.trie[parts] = HashInfo.from_dict(entry)
+        return ret
+
+    def to_list(self):
+        # Sorting the list by path to ensure reproducibility
+        return sorted(
+            (
+                {
+                    # NOTE: not using hash_info.to_dict() because we don't want
+                    # size/nfiles fields at this point.
+                    hash_info.name: hash_info.value,
+                    self.PARAM_RELPATH: posixpath.sep.join(parts),
+                }
+                for parts, hash_info in self.trie.iteritems()  # noqa: B301
+            ),
+            key=itemgetter(self.PARAM_RELPATH),
+        )
+
+    def merge(self, ancestor, their):
+        merged = DirInfo()
+        merged.trie = _merge(ancestor.trie, self.trie, their.trie)
+        return merged

dvc/dvcfile.py

@@ -2,12 +2,10 @@ import collections
 import contextlib
 import logging
 import os
-from typing import TYPE_CHECKING, Any, Union
 
 from voluptuous import MultipleInvalid
 
 from dvc.exceptions import DvcException
-from dvc.parsing.versions import LOCKFILE_VERSION, SCHEMA_KWD
 from dvc.stage import serialize
 from dvc.stage.exceptions import (
     StageFileBadNameError,
@@ -15,19 +13,17 @@ from dvc.stage.exceptions import (
     StageFileFormatError,
     StageFileIsNotDvcFileError,
 )
-from dvc.types import AnyPath
+from dvc.stage.loader import SingleStageLoader, StageLoader
 from dvc.utils import relpath
 from dvc.utils.collections import apply_diff
 from dvc.utils.serialize import (
     dump_yaml,
+    load_yaml,
     modify_yaml,
     parse_yaml,
     parse_yaml_for_update,
 )
 
-if TYPE_CHECKING:
-    from dvc.repo import Repo
-
 logger = logging.getLogger(__name__)
 
 DVC_FILE = "Dvcfile"
@@ -36,11 +32,6 @@ PIPELINE_FILE = "dvc.yaml"
 PIPELINE_LOCK = "dvc.lock"
 
 
-class FileIsGitIgnored(DvcException):
-    def __init__(self, path):
-        super().__init__(f"'{path}' is git-ignored.")
-
-
 class LockfileCorruptedError(DvcException):
     pass
 
@@ -69,9 +60,9 @@ def is_lock_file(path):
 def check_dvc_filename(path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
-            "bad DVC file name '{}'. DVC files should be named "
-            "'{}' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
-                relpath(path), PIPELINE_FILE, os.path.basename(path)
+            "bad DVC-file name '{}'. DVC-files should be named "
+            "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
+                relpath(path), os.path.basename(path)
             )
         )
 
@@ -105,48 +96,31 @@ class FileMixin:
         return relpath(self.path)
 
     def exists(self):
-        return self.repo.fs.exists(self.path)
-
-    def _is_git_ignored(self):
-        from dvc.fs.local import LocalFileSystem
-
-        return isinstance(
-            self.repo.fs, LocalFileSystem
-        ) and self.repo.scm.is_ignored(self.path)
-
-    def _verify_filename(self):
-        if self.verify:
-            check_dvc_filename(self.path)
-
-    def _check_gitignored(self):
-        if self._is_git_ignored():
-            raise FileIsGitIgnored(self.path)
+        return self.repo.tree.exists(self.path)
 
     def _load(self):
         # it raises the proper exceptions by priority:
         # 1. when the file doesn't exists
-        # 2. filename is not a DVC file
+        # 2. filename is not a DVC-file
         # 3. path doesn't represent a regular file
-        # 4. when the file is git ignored
         if not self.exists():
-            is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
-            raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
-
-        self._verify_filename()
-        if not self.repo.fs.isfile(self.path):
+            raise StageFileDoesNotExistError(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
+        if not self.repo.tree.isfile(self.path):
             raise StageFileIsNotDvcFileError(self.path)
 
-        self._check_gitignored()
-        with self.repo.fs.open(self.path, encoding="utf-8") as fd:
+        with self.repo.tree.open(self.path) as fd:
             stage_text = fd.read()
         d = parse_yaml(stage_text, self.path)
-        return self.validate(d, self.relpath), stage_text
+        self.validate(d, self.relpath)
+        return d, stage_text
 
     @classmethod
     def validate(cls, d, fname=None):
         assert isinstance(cls.SCHEMA, collections.abc.Callable)
         try:
-            return cls.SCHEMA(d)  # pylint: disable=not-callable
+            cls.SCHEMA(d)  # pylint: disable=not-callable
         except MultipleInvalid as exc:
             raise StageFileFormatError(f"'{fname}' format error: {exc}")
 
@@ -163,17 +137,16 @@ class FileMixin:
 
 class SingleStageFile(FileMixin):
     from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
-    from dvc.stage.loader import SingleStageLoader as LOADER
 
     @property
     def stage(self):
         data, raw = self._load()
-        return self.LOADER.load_stage(self, data, raw)
+        return SingleStageLoader.load_stage(self, data, raw)
 
     @property
     def stages(self):
         data, raw = self._load()
-        return self.LOADER(self, data, raw)
+        return SingleStageLoader(self, data, raw)
 
     def dump(self, stage, **kwargs):
         """Dumps given stage appropriately in the dvcfile."""
@@ -204,7 +177,6 @@ class PipelineFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
 
     from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
-    from dvc.stage.loader import StageLoader as LOADER
 
     @property
     def _lockfile(self):
@@ -238,7 +210,7 @@ class PipelineFile(FileMixin):
         self._check_if_parametrized(stage)
         stage_data = serialize.to_pipeline_file(stage)
 
-        with modify_yaml(self.path, fs=self.repo.fs) as data:
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
             if not data:
                 logger.info("Creating '%s'", self.relpath)
 
@@ -267,7 +239,7 @@ class PipelineFile(FileMixin):
     def stages(self):
         data, _ = self._load()
         lockfile_data = self._lockfile.load()
-        return self.LOADER(self, data, lockfile_data)
+        return StageLoader(self, data, lockfile_data)
 
     def remove(self, force=False):
         if not force:
@@ -282,7 +254,7 @@ class PipelineFile(FileMixin):
         if not self.exists():
             return
 
-        with open(self.path, "r", encoding="utf-8") as f:
+        with open(self.path, "r") as f:
             d = parse_yaml_for_update(f.read(), self.path)
 
         self.validate(d, self.path)
@@ -301,90 +273,35 @@ class PipelineFile(FileMixin):
         raise NotImplementedError
 
 
-def get_lockfile_schema(d):
-    from dvc.schema import (
-        COMPILED_LOCKFILE_V1_SCHEMA,
-        COMPILED_LOCKFILE_V2_SCHEMA,
-    )
-
-    schema = {
-        LOCKFILE_VERSION.V1: COMPILED_LOCKFILE_V1_SCHEMA,
-        LOCKFILE_VERSION.V2: COMPILED_LOCKFILE_V2_SCHEMA,
-    }
-
-    version = LOCKFILE_VERSION.from_dict(d)
-    return schema[version]
-
-
-def migrate_lock_v1_to_v2(d, version_info):
-    stages = {k: v for k, v in d.items()}
-
-    for key in stages:
-        d.pop(key)
-
-    # forcing order, meta should always be at the top
-    d.update(version_info)
-    d["stages"] = stages
-
-
 class Lockfile(FileMixin):
-    @classmethod
-    def validate(cls, d, fname=None):
-        schema = get_lockfile_schema(d)
-        try:
-            return schema(d)
-        except MultipleInvalid as exc:
-            raise StageFileFormatError(f"'{fname}' format error: {exc}")
-
-    def _verify_filename(self):
-        pass  # lockfile path is hardcoded, so no need to verify here
-
-    def _load(self):
-        try:
-            return super()._load()
-        except StageFileDoesNotExistError:
-            # we still need to account for git-ignored dvc.lock file
-            # even though it may not exist or have been .dvcignored
-            self._check_gitignored()
-            return {}, ""
-        except StageFileFormatError as exc:
-            raise LockfileCorruptedError(
-                f"Lockfile '{self.relpath}' is corrupted."
-            ) from exc
+    from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
 
     def load(self):
-        d, _ = self._load()
-        return d
+        if not self.exists():
+            return {}
 
-    @property
-    def latest_version_info(self):
-        version = LOCKFILE_VERSION.V2.value  # pylint:disable=no-member
-        return {SCHEMA_KWD: version}
+        data = load_yaml(self.path, tree=self.repo.tree)
+        try:
+            self.validate(data, fname=self.relpath)
+        except StageFileFormatError:
+            raise LockfileCorruptedError(
+                f"Lockfile '{self.relpath}' is corrupted."
+            )
+        return data
 
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage)
 
-        with modify_yaml(self.path, fs=self.repo.fs) as data:
-            version = LOCKFILE_VERSION.from_dict(data)
-            if version == LOCKFILE_VERSION.V1:
-                logger.info(
-                    "Migrating lock file '%s' from v1 to v2", self.relpath
-                )
-                migrate_lock_v1_to_v2(data, self.latest_version_info)
-            else:
-                if not data:
-                    data.update(self.latest_version_info)
-                    # order is important, meta should always be at the top
-                    logger.info("Generating lock file '%s'", self.relpath)
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
+            if not data:
+                logger.info("Generating lock file '%s'", self.relpath)
 
-            data["stages"] = data.get("stages", {})
-            modified = data["stages"].get(stage.name, {}) != stage_data.get(
+            modified = data.get(stage.name, {}) != stage_data.get(
                 stage.name, {}
             )
             if modified:
                 logger.info("Updating lock file '%s'", self.relpath)
-
-            data["stages"].update(stage_data)
+            data.update(stage_data)
 
         if modified:
             self.repo.scm.track_file(self.relpath)
@@ -393,19 +310,17 @@ class Lockfile(FileMixin):
         if not self.exists():
             return
 
-        with open(self.path, encoding="utf-8") as f:
+        with open(self.path) as f:
             d = parse_yaml_for_update(f.read(), self.path)
         self.validate(d, self.path)
 
-        version = LOCKFILE_VERSION.from_dict(d)
-        data = d if version == LOCKFILE_VERSION.V1 else d.get("stages", {})
-        if stage.name not in data:
+        if stage.name not in d:
             return
 
         logger.debug("Removing '%s' from '%s'", stage.name, self.path)
-        del data[stage.name]
+        del d[stage.name]
 
-        if data:
+        if d:
             dump_yaml(self.path, d)
         else:
             self.remove()
@@ -415,19 +330,12 @@ class Lockfile(FileMixin):
 
 
 class Dvcfile:
-    def __new__(cls, repo: "Repo", path: AnyPath, **kwargs: Any):
+    def __new__(cls, repo, path, **kwargs):
         assert path
         assert repo
 
-        return make_dvcfile(repo, path, **kwargs)
-
-
-DVCFile = Union["PipelineFile", "SingleStageFile"]
-
-
-def make_dvcfile(repo: "Repo", path: AnyPath, **kwargs: Any) -> DVCFile:
-    _, ext = os.path.splitext(str(path))
-    if ext in [".yaml", ".yml"]:
-        return PipelineFile(repo, path, **kwargs)
-    # fallback to single stage file for better error messages
-    return SingleStageFile(repo, path, **kwargs)
+        _, ext = os.path.splitext(path)
+        if ext in [".yaml", ".yml"]:
+            return PipelineFile(repo, path, **kwargs)
+        # fallback to single stage file for better error messages
+        return SingleStageFile(repo, path, **kwargs)

dvc/env.py

@@ -2,7 +2,3 @@ DVC_CHECKPOINT = "DVC_CHECKPOINT"
 DVC_DAEMON = "DVC_DAEMON"
 DVC_PAGER = "DVC_PAGER"
 DVC_ROOT = "DVC_ROOT"
-DVCLIVE_PATH = "DVCLIVE_PATH"
-DVCLIVE_SUMMARY = "DVCLIVE_SUMMARY"
-DVCLIVE_HTML = "DVCLIVE_HTML"
-DVCLIVE_RESUME = "DVCLIVE_RESUME"

dvc/exceptions.py

@@ -1,5 +1,7 @@
 """Exceptions raised by the dvc."""
-from typing import List
+from funcy import first
+
+from dvc.utils import error_link, format_link, relpath
 
 
 class DvcException(Exception):
@@ -7,7 +9,6 @@ class DvcException(Exception):
 
     def __init__(self, msg, *args):
         assert msg
-        self.msg = msg
         super().__init__(msg, *args)
 
 
@@ -25,8 +26,6 @@ class OutputDuplicationError(DvcException):
     """
 
     def __init__(self, output, stages):
-        from funcy import first
-
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
         if len(stages) == 1:
@@ -50,12 +49,10 @@ class OutputNotFoundError(DvcException):
     """
 
     def __init__(self, output, repo=None):
-        from dvc.utils import relpath
-
         self.output = output
         self.repo = repo
         super().__init__(
-            "Unable to find DVC file with output '{path}'".format(
+            "Unable to find DVC-file with output '{path}'".format(
                 path=relpath(self.output)
             )
         )
@@ -173,11 +170,11 @@ class BadMetricError(DvcException):
         )
 
 
-class MetricsError(DvcException):
+class NoMetricsError(DvcException):
     pass
 
 
-class NoMetricsParsedError(MetricsError):
+class NoMetricsParsedError(NoMetricsError):
     def __init__(self, command):
         super().__init__(
             f"Could not parse {command} files. Use `-v` option to see more "
@@ -185,7 +182,7 @@ class NoMetricsParsedError(MetricsError):
         )
 
 
-class NoMetricsFoundError(MetricsError):
+class NoMetricsFoundError(NoMetricsError):
     def __init__(self, command, run_options):
         super().__init__(
             f"No {command} files in this repository. "
@@ -194,15 +191,6 @@ class NoMetricsFoundError(MetricsError):
         )
 
 
-class MetricDoesNotExistError(MetricsError):
-    def __init__(self, targets: List[str]):
-        if len(targets) == 1:
-            msg = "'{}' does not exist."
-        else:
-            msg = "'{}' do not exist."
-        super().__init__(msg.format(", ".join(targets)))
-
-
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super().__init__(
@@ -239,6 +227,11 @@ class FileMissingError(DvcException):
         )
 
 
+class FileOwnershipError(DvcException):
+    def __init__(self, path):
+        super().__init__(f"file '{path}' not owned by user! ")
+
+
 class DvcIgnoreInCollectedDirError(DvcException):
     def __init__(self, ignore_dirname):
         super().__init__(
@@ -249,8 +242,6 @@ class DvcIgnoreInCollectedDirError(DvcException):
 
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
-        from dvc.utils import format_link
-
         super().__init__(
             "Hook '{}' already exists. Please refer to {} for more "
             "info.".format(
@@ -275,8 +266,6 @@ class UploadError(DvcException):
 
 class CheckoutError(DvcException):
     def __init__(self, target_infos, stats=None):
-        from dvc.utils import error_link
-
         self.target_infos = target_infos
         self.stats = stats
         targets = [str(t) for t in target_infos]
@@ -302,8 +291,6 @@ class NoRemoteInExternalRepoError(DvcException):
 
 class NoOutputInExternalRepoError(DvcException):
     def __init__(self, path, external_repo_path, external_repo_url):
-        from dvc.utils import relpath
-
         super().__init__(
             "Output '{}' not found in target repository '{}'".format(
                 relpath(path, external_repo_path), external_repo_url
@@ -334,8 +321,6 @@ class PathMissingError(DvcException):
 
 class RemoteCacheRequiredError(DvcException):
     def __init__(self, path_info):
-        from dvc.utils import format_link
-
         super().__init__(
             (
                 "Current operation was unsuccessful because '{}' requires "
@@ -370,8 +355,6 @@ class MergeError(DvcException):
 
 
 class CacheLinkError(DvcException):
-    from dvc.utils import format_link
-
     SUPPORT_LINK = "See {} for more information.".format(
         format_link(
             "https://dvc.org/doc/user-guide/troubleshooting#cache-types"

dvc/external_repo.py

@@ -3,31 +3,39 @@ import os
 import tempfile
 import threading
 from contextlib import contextmanager
-from typing import Dict
+from distutils.dir_util import copy_tree
+from typing import Dict, Iterable
 
-from funcy import retry, wrap_with
+from funcy import cached_property, reraise, retry, wrap_with
 
+from dvc.cache import Cache
+from dvc.config import NoRemoteError, NotDvcRepoError
 from dvc.exceptions import (
+    DvcException,
     FileMissingError,
     NoOutputInExternalRepoError,
     NoRemoteInExternalRepoError,
-    NotDvcRepoError,
     OutputNotFoundError,
     PathMissingError,
 )
+from dvc.path_info import PathInfo
 from dvc.repo import Repo
+from dvc.scm.base import CloneError
+from dvc.scm.git import Git
+from dvc.tree.local import LocalTree
+from dvc.tree.repo import RepoTree
 from dvc.utils import relpath
+from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
 
-@contextmanager
-def external_repo(
-    url, rev=None, for_write=False, cache_dir=None, cache_types=None, **kwargs
-):
-    from dvc.config import NoRemoteError
-    from dvc.scm.git import Git
+class IsADVCRepoError(DvcException):
+    """Raised when it is not expected to be a dvc repo."""
 
+
+@contextmanager
+def external_repo(url, rev=None, for_write=False, **kwargs):
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
     # Local HEAD points to the tip of whatever branch we first cloned from
@@ -35,42 +43,17 @@ def external_repo(
     # the tip of the default branch
     rev = rev or "refs/remotes/origin/HEAD"
 
-    cache_config = {
-        "cache": {
-            "dir": cache_dir or _get_cache_dir(url),
-            "type": cache_types,
-        }
-    }
-
-    config = _get_remote_config(url) if os.path.isdir(url) else {}
-    config.update(cache_config)
-
-    def make_repo(path, **_kwargs):
-        _config = cache_config.copy()
-        if os.path.isdir(url):
-            rel = os.path.relpath(path, _kwargs["scm"].root_dir)
-            repo_path = os.path.join(url, rel)
-            _config.update(_get_remote_config(repo_path))
-        return Repo(path, config=_config, **_kwargs)
-
     root_dir = path if for_write else os.path.realpath(path)
-    repo_kwargs = dict(
+    conf = dict(
         root_dir=root_dir,
         url=url,
         scm=None if for_write else Git(root_dir),
         rev=None if for_write else rev,
-        config=config,
-        repo_factory=make_repo,
+        for_write=for_write,
+        uninitialized=True,
         **kwargs,
     )
-
-    if "subrepos" not in repo_kwargs:
-        repo_kwargs["subrepos"] = True
-
-    if "uninitialized" not in repo_kwargs:
-        repo_kwargs["uninitialized"] = True
-
-    repo = Repo(**repo_kwargs)
+    repo = ExternalRepo(**conf)
 
     try:
         yield repo
@@ -94,15 +77,6 @@ CLONES: Dict[str, str] = {}
 CACHE_DIRS: Dict[str, str] = {}
 
 
-@wrap_with(threading.Lock())
-def _get_cache_dir(url):
-    try:
-        cache_dir = CACHE_DIRS[url]
-    except KeyError:
-        cache_dir = CACHE_DIRS[url] = tempfile.mkdtemp("dvc-cache")
-    return cache_dir
-
-
 def clean_repos():
     # Outside code should not see cache while we are removing
     paths = [path for path, _ in CLONES.values()] + list(CACHE_DIRS.values())
@@ -113,28 +87,206 @@ def clean_repos():
         _remove(path)
 
 
-def _get_remote_config(url):
-    try:
-        repo = Repo(url)
-    except NotDvcRepoError:
-        return {}
+class ExternalRepo(Repo):
+    # pylint: disable=no-member
 
-    try:
-        name = repo.config["core"].get("remote")
-        if not name:
-            # Fill the empty upstream entry with a new remote pointing to the
-            # original repo's cache location.
-            name = "auto-generated-upstream"
-            return {
-                "core": {"remote": name},
-                "remote": {name: {"url": repo.odb.local.cache_dir}},
-            }
+    def __init__(
+        self,
+        root_dir,
+        url,
+        scm=None,
+        rev=None,
+        for_write=False,
+        cache_dir=None,
+        cache_types=None,
+        uninitialized=False,
+        **kwargs,
+    ):
+        super().__init__(
+            root_dir, scm=scm, rev=rev, uninitialized=uninitialized
+        )
 
-        # Use original remote to make sure that we are using correct url,
-        # credential paths, etc if they are relative to the config location.
-        return {"remote": {name: repo.config["remote"][name]}}
-    finally:
-        repo.close()
+        self.url = url
+        self.for_write = for_write
+        self.cache_dir = cache_dir or self._get_cache_dir()
+        self.cache_types = cache_types
+
+        self._setup_cache(self)
+        self._fix_upstream(self)
+        self.tree_confs = kwargs
+
+    def __str__(self):
+        return self.url
+
+    @cached_property
+    def repo_tree(self):
+        return self._get_tree_for(
+            self, subrepos=not self.for_write, repo_factory=self.make_repo
+        )
+
+    def get_rev(self):
+        assert self.scm
+        if isinstance(self.tree, LocalTree):
+            return self.scm.get_rev()
+        return self.tree.rev
+
+    def _fetch_to_cache(self, path_info, repo, callback, **kwargs):
+        # don't support subrepo traversal as it might fail due to difference
+        # in remotes
+        tree = self._get_tree_for(repo)
+        cache = repo.cache.local
+
+        hash_info = tree.get_hash(
+            path_info, download_callback=callback, **kwargs
+        )
+        cache.save(
+            path_info,
+            tree,
+            hash_info,
+            save_link=False,
+            download_callback=callback,
+        )
+        return hash_info
+
+    def fetch_external(self, paths: Iterable, **kwargs):
+        """Fetch specified external repo paths into cache.
+
+        Returns 3-tuple in the form
+            (downloaded, failed, list(cache_infos))
+        where cache_infos can be used as checkout targets for the
+        fetched paths.
+        """
+        download_results = []
+        failed = 0
+        root = PathInfo(self.root_dir)
+
+        paths = [root / path for path in paths]
+
+        def download_update(result):
+            download_results.append(result)
+
+        hash_infos = []
+        for path in paths:
+            with reraise(FileNotFoundError, PathMissingError(path, self.url)):
+                metadata = self.repo_tree.metadata(path)
+
+            self._check_repo(path, metadata.repo)
+            repo = metadata.repo
+            hash_info = self._fetch_to_cache(
+                path, repo, download_update, **kwargs
+            )
+            hash_infos.append(hash_info)
+
+        return sum(download_results), failed, hash_infos
+
+    def _check_repo(self, path_info, repo):
+        if not repo:
+            return
+
+        repo_path = PathInfo(repo.root_dir)
+        if path_info == repo_path and isinstance(repo, Repo):
+            message = "Cannot fetch a complete DVC repository"
+            if repo.root_dir != self.root_dir:
+                rel = relpath(repo.root_dir, self.root_dir)
+                message += f" '{rel}'"
+            raise IsADVCRepoError(message)
+
+    def get_external(self, path, dest):
+        """Convenience wrapper for fetch_external and checkout."""
+        path_info = PathInfo(self.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, self.url)):
+            metadata = self.repo_tree.metadata(path_info)
+
+        self._check_repo(path_info, metadata.repo)
+        if metadata.output_exists:
+            repo = metadata.repo
+            cache = repo.cache.local
+            # fetch DVC and git files to tmpdir cache, then checkout
+            save_info = self._fetch_to_cache(path_info, repo, None)
+            cache.checkout(PathInfo(dest), save_info)
+        else:
+            # git-only folder, just copy files directly to dest
+            tree = self._get_tree_for(metadata.repo)  # ignore subrepos
+            tree.copytree(path_info, dest)
+
+    def _get_tree_for(self, repo, **kwargs):
+        """
+        Provides a combined tree of a single repo with dvc + git/local tree.
+        """
+        kw = {**self.tree_confs, **kwargs}
+        if "fetch" not in kw:
+            kw["fetch"] = True
+        return RepoTree(repo, **kw)
+
+    def get_checksum(self, path):
+        path_info = PathInfo(self.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, self.url)):
+            metadata = self.repo_tree.metadata(path_info)
+
+        # skip subrepos to check for
+        tree = self._get_tree_for(metadata.repo)
+        return tree.get_hash(path_info)
+
+    @staticmethod
+    def _fix_local_remote(orig_repo, src_repo, remote_name):
+        # If a remote URL is relative to the source repo,
+        # it will have changed upon config load and made
+        # relative to this new repo. Restore the old one here.
+        new_remote = orig_repo.config["remote"][remote_name]
+        old_remote = src_repo.config["remote"][remote_name]
+        if new_remote["url"] != old_remote["url"]:
+            new_remote["url"] = old_remote["url"]
+
+    @staticmethod
+    def _add_upstream(orig_repo, src_repo):
+        # Fill the empty upstream entry with a new remote pointing to the
+        # original repo's cache location.
+        cache_dir = src_repo.cache.local.cache_dir
+        orig_repo.config["remote"]["auto-generated-upstream"] = {
+            "url": cache_dir
+        }
+        orig_repo.config["core"]["remote"] = "auto-generated-upstream"
+
+    def make_repo(self, path):
+        repo = Repo(path, scm=self.scm, rev=self.get_rev())
+
+        self._setup_cache(repo)
+        self._fix_upstream(repo)
+
+        return repo
+
+    def _setup_cache(self, repo):
+        repo.config["cache"]["dir"] = self.cache_dir
+        repo.cache = Cache(repo)
+        if self.cache_types:
+            repo.cache.local.cache_types = self.cache_types
+
+    def _fix_upstream(self, repo):
+        if not os.path.isdir(self.url):
+            return
+
+        try:
+            rel_path = os.path.relpath(repo.root_dir, self.root_dir)
+            src_repo = Repo(PathInfo(self.url) / rel_path)
+        except NotDvcRepoError:
+            return
+
+        try:
+            remote_name = repo.config["core"].get("remote")
+            if remote_name:
+                self._fix_local_remote(repo, src_repo, remote_name)
+            else:
+                self._add_upstream(repo, src_repo)
+        finally:
+            src_repo.close()
+
+    @wrap_with(threading.Lock())
+    def _get_cache_dir(self):
+        try:
+            cache_dir = CACHE_DIRS[self.url]
+        except KeyError:
+            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
+        return cache_dir
 
 
 def _cached_clone(url, rev, for_write=False):
@@ -144,8 +296,6 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    from distutils.dir_util import copy_tree
-
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
     clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
@@ -172,8 +322,6 @@ def _clone_default_branch(url, rev, for_write=False):
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
-    from dvc.scm.git import Git
-
     clone_path, shallow = CLONES.get(url, (None, False))
 
     git = None
@@ -201,8 +349,6 @@ def _clone_default_branch(url, rev, for_write=False):
             clone_path = tempfile.mkdtemp("dvc-clone")
             if not for_write and rev and not Git.is_sha(rev):
                 # If rev is a tag or branch name try shallow clone first
-                from dvc.scm.base import CloneError
-
                 try:
                     git = Git.clone(url, clone_path, shallow_branch=rev)
                     shallow = True
@@ -223,21 +369,19 @@ def _clone_default_branch(url, rev, for_write=False):
 
 
 def _unshallow(git):
-    if git.gitpython.repo.head.is_detached:
+    if git.repo.head.is_detached:
         # If this is a detached head (i.e. we shallow cloned a tag) switch to
         # the default branch
-        origin_refs = git.gitpython.repo.remotes["origin"].refs
+        origin_refs = git.repo.remotes["origin"].refs
         ref = origin_refs["HEAD"].reference
         branch_name = ref.name.split("/")[-1]
-        branch = git.gitpython.repo.create_head(branch_name, ref)
+        branch = git.repo.create_head(branch_name, ref)
         branch.set_tracking_branch(ref)
         branch.checkout()
     git.pull(unshallow=True)
 
 
 def _git_checkout(repo_path, rev):
-    from dvc.scm.git import Git
-
     logger.debug("erepo: git checkout %s@%s", repo_path, rev)
     git = Git(repo_path)
     try:
@@ -247,8 +391,6 @@ def _git_checkout(repo_path, rev):
 
 
 def _remove(path):
-    from dvc.utils.fs import remove
-
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)

dvc/fs/azure.py

@@ -1,125 +0,0 @@
-import logging
-import os
-import threading
-
-from funcy import cached_property, wrap_prop
-
-from dvc.exceptions import DvcException
-from dvc.path_info import CloudURLInfo
-from dvc.scheme import Schemes
-from dvc.utils import format_link
-
-from .fsspec_wrapper import FSSpecWrapper
-
-logger = logging.getLogger(__name__)
-_DEFAULT_CREDS_STEPS = (
-    "https://azuresdkdocs.blob.core.windows.net/$web/python/"
-    "azure-identity/1.4.0/azure.identity.html#azure.identity"
-    ".DefaultAzureCredential"
-)
-
-
-class AzureAuthError(DvcException):
-    pass
-
-
-class AzureFileSystem(FSSpecWrapper):
-    scheme = Schemes.AZURE
-    PATH_CLS = CloudURLInfo
-    PARAM_CHECKSUM = "etag"
-    DETAIL_FIELDS = frozenset(("etag", "size"))
-    REQUIRES = {
-        "adlfs": "adlfs",
-        "knack": "knack",
-        "azure-identity": "azure.identity",
-    }
-
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-
-        url = config.get("url")
-        self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
-
-        if not self.bucket:
-            container = self._az_config.get("storage", "container_name", None)
-            url = f"azure://{container}"
-
-        self.path_info = self.PATH_CLS(url)
-        self.bucket = self.path_info.bucket
-
-    def _prepare_credentials(self, config):
-        from azure.identity.aio import DefaultAzureCredential
-
-        login_info = {}
-        login_info["connection_string"] = config.get(
-            "connection_string",
-            self._az_config.get("storage", "connection_string", None),
-        )
-        login_info["account_name"] = config.get(
-            "account_name", self._az_config.get("storage", "account", None)
-        )
-        login_info["account_key"] = config.get(
-            "account_key", self._az_config.get("storage", "key", None)
-        )
-        login_info["sas_token"] = config.get(
-            "sas_token", self._az_config.get("storage", "sas_token", None)
-        )
-        login_info["tenant_id"] = config.get("tenant_id")
-        login_info["client_id"] = config.get("client_id")
-        login_info["client_secret"] = config.get("client_secret")
-
-        if not any(login_info.values()):
-            login_info["credentials"] = DefaultAzureCredential()
-
-        for login_method, required_keys in [  # noqa
-            ("connection string", ["connection_string"]),
-            (
-                "AD service principal",
-                ["tenant_id", "client_id", "client_secret"],
-            ),
-            ("account key", ["account_name", "account_key"]),
-            ("SAS token", ["account_name", "sas_token"]),
-            ("anonymous login", ["account_name"]),
-            (f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
-        ]:
-            if all(login_info.get(key) is not None for key in required_keys):
-                break
-        else:
-            login_method = None
-
-        self.login_method = login_method
-        return login_info
-
-    @cached_property
-    def _az_config(self):
-        # NOTE: ideally we would've used get_default_cli().config from
-        # azure.cli.core, but azure-cli-core has a lot of conflicts with other
-        # dependencies. So instead we are just use knack directly
-        from knack.config import CLIConfig
-
-        config_dir = os.getenv(
-            "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
-        )
-        return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
-
-    @wrap_prop(threading.Lock())
-    @cached_property
-    def fs(self):
-        from adlfs import AzureBlobFileSystem
-        from azure.core.exceptions import AzureError
-
-        try:
-            file_system = AzureBlobFileSystem(**self.fs_args)
-            if self.bucket not in [
-                container.rstrip("/") for container in file_system.ls("/")
-            ]:
-                file_system.mkdir(self.bucket)
-        except (ValueError, AzureError) as e:
-            raise AzureAuthError(
-                f"Authentication to Azure Blob Storage via {self.login_method}"
-                " failed.\nLearn more about configuration settings at"
-                f" {format_link('https://man.dvc.org/remote/modify')}"
-            ) from e
-
-        return file_system

dvc/fs/fsspec_wrapper.py

@@ -1,158 +0,0 @@
-import os
-import shutil
-from functools import lru_cache
-
-from dvc.progress import Tqdm
-
-from .base import BaseFileSystem
-
-
-# pylint: disable=no-member
-class FSSpecWrapper(BaseFileSystem):
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-        self.fs_args = {"skip_instance_cache": True}
-        self.fs_args.update(self._prepare_credentials(config))
-
-    def fs(self):
-        raise NotImplementedError
-
-    @lru_cache(512)
-    def _with_bucket(self, path):
-        if isinstance(path, self.PATH_CLS):
-            return f"{path.bucket}/{path.path}"
-        return path
-
-    def _strip_bucket(self, entry):
-        try:
-            bucket, path = entry.split("/", 1)
-        except ValueError:
-            # If there is no path attached, only returns
-            # the bucket (top-level).
-            bucket, path = entry, None
-
-        return path or bucket
-
-    def _strip_buckets(self, entries, detail, prefix=None):
-        for entry in entries:
-            if detail:
-                entry = self._entry_hook(entry.copy())
-                entry["name"] = self._strip_bucket(entry["name"])
-            else:
-                entry = self._strip_bucket(
-                    f"{prefix}/{entry}" if prefix else entry
-                )
-            yield entry
-
-    def _entry_hook(self, entry):
-        """Simple hook method to be overriden when wanted to process
-        entries within info() and ls(detail=True) calls"""
-        return entry
-
-    def _prepare_credentials(self, config):  # pylint: disable=unused-argument
-        """Prepare the arguments for authentication to the
-        host filesystem"""
-        return {}
-
-    def isdir(self, path_info):
-        if not self.exists(path_info):
-            return False
-
-        # Directory in object storages are interpreted differently
-        # among different fsspec providers, so this logic is a temporary
-        # measure for us to adapt as of now. It checks whether it is a
-        # directory (as in a prefix with contents) or whether it is an empty
-        # file where it's name ends with a forward slash
-        entry = self.info(path_info)
-        return entry["type"] == "directory" or (
-            entry["size"] == 0
-            and entry["type"] == "file"
-            and entry["name"].endswith("/")
-        )
-
-    def isfile(self, path_info):
-        if not self.exists(path_info):
-            return False
-
-        return not self.isdir(path_info)
-
-    def is_empty(self, path_info):
-        entry = self.info(path_info)
-        if entry["type"] == "directory":
-            return not self.fs.ls(self._with_bucket(path_info))
-        return entry["size"] == 0
-
-    def open(
-        self, path_info, mode="r", **kwargs
-    ):  # pylint: disable=arguments-differ
-        return self.fs.open(self._with_bucket(path_info), mode=mode)
-
-    def copy(self, from_info, to_info):
-        self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
-
-    def exists(self, path_info, use_dvcignore=False):
-        return self.fs.exists(self._with_bucket(path_info))
-
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        if self.isdir(path_info) and self.is_empty(path_info):
-            return None
-
-        path = self._with_bucket(path_info)
-        if recursive:
-            for root, _, files in self.fs.walk(path, detail=detail):
-                if detail:
-                    files = files.values()
-                yield from self._strip_buckets(files, detail, prefix=root)
-            return None
-
-        yield from self._strip_buckets(self.ls(path, detail=detail), detail)
-
-    def walk_files(self, path_info, **kwargs):
-        for file in self.ls(path_info, recursive=True):
-            yield path_info.replace(path=file)
-
-    def remove(self, path_info):
-        self.fs.rm(self._with_bucket(path_info))
-
-    def info(self, path_info):
-        info = self.fs.info(self._with_bucket(path_info)).copy()
-        info = self._entry_hook(info)
-        info["name"] = self._strip_bucket(info["name"])
-        return info
-
-    def _upload_fobj(self, fobj, to_info):
-        with self.open(to_info, "wb") as fdest:
-            shutil.copyfileobj(fobj, fdest, length=fdest.blocksize)
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **kwargs
-    ):
-        total = os.path.getsize(from_file)
-        with open(from_file, "rb") as fobj:
-            self.upload_fobj(
-                fobj,
-                self._with_bucket(to_info),
-                desc=name,
-                total=total,
-                no_progress_bar=no_progress_bar,
-            )
-        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
-
-    def _download(
-        self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
-    ):
-        total = self.getsize(self._with_bucket(from_info))
-        with self.open(from_info, "rb") as fobj:
-            with Tqdm.wrapattr(
-                fobj,
-                "read",
-                desc=name,
-                disable=no_progress_bar,
-                bytes=True,
-                total=total,
-                **pbar_args,
-            ) as wrapped:
-                with open(to_file, "wb") as fdest:
-                    shutil.copyfileobj(wrapped, fdest, length=fobj.blocksize)

dvc/fs/git.py

@@ -1,168 +0,0 @@
-import errno
-import os
-
-from funcy import cached_property
-
-from dvc.utils import is_exec, relpath
-
-from .base import BaseFileSystem
-
-
-class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
-    """Proxies the repo file access methods to Git objects"""
-
-    def __init__(
-        self, root_dir, trie, use_dvcignore=False, dvcignore_root=None
-    ):
-        super().__init__(None, {})
-        self._fs_root = root_dir
-        self.trie = trie
-        self.use_dvcignore = use_dvcignore
-        self.dvcignore_root = dvcignore_root
-
-    @property
-    def rev(self):
-        return self.trie.rev
-
-    @property
-    def fs_root(self):
-        return self._fs_root
-
-    def _get_key(self, path):
-        if isinstance(path, str):
-            if not os.path.isabs(path):
-                relparts = path.split(os.sep)
-            else:
-                relparts = relpath(path, self.fs_root).split(os.sep)
-        else:
-            relparts = path.relative_to(self.fs_root).parts
-        if relparts == ["."]:
-            return ()
-        return tuple(relparts)
-
-    @cached_property
-    def dvcignore(self):
-        from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
-
-        root = self.dvcignore_root or self.fs_root
-        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
-        return cls(self, root)
-
-    def open(
-        self, path, mode="r", encoding=None
-    ):  # pylint: disable=arguments-differ
-        # NOTE: this is incorrect, we need to use locale to determine default
-        # encoding.
-        encoding = encoding or "utf-8"
-
-        key = self._get_key(path)
-        try:
-            return self.trie.open(key, mode=mode, encoding=encoding)
-        except KeyError as exc:
-            msg = os.strerror(errno.ENOENT) + f"in branch '{self.rev}'"
-            raise FileNotFoundError(errno.ENOENT, msg, path) from exc
-        except IsADirectoryError as exc:
-            raise IsADirectoryError(
-                errno.EISDIR, os.strerror(errno.EISDIR), path
-            ) from exc
-
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        def _is_ignored(path):
-            return self.dvcignore.is_ignored_file(
-                path
-            ) or self.dvcignore.is_ignored_dir(path)
-
-        if use_dvcignore and _is_ignored(path):
-            return False
-
-        key = self._get_key(path)
-        return self.trie.exists(key)
-
-    def isdir(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        if use_dvcignore and self.dvcignore.is_ignored_dir(path):
-            return False
-        key = self._get_key(path)
-        return self.trie.isdir(key)
-
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        if self.dvcignore.is_ignored_file(path):
-            return False
-        key = self._get_key(path)
-        return self.trie.isfile(key)
-
-    def walk(
-        self,
-        top,
-        topdown=True,
-        onerror=None,
-        use_dvcignore=True,
-        ignore_subrepos=True,
-    ):
-        """Directory tree generator.
-
-        See `os.walk` for the docs. Differences:
-        - no support for symlinks
-        """
-        if not self.isdir(top, use_dvcignore=use_dvcignore):
-            if onerror:
-                if self.exists(top, use_dvcignore=use_dvcignore):
-                    exc = NotADirectoryError(
-                        errno.ENOTDIR, os.strerror(errno.ENOTDIR), top
-                    )
-                else:
-                    exc = FileNotFoundError(
-                        errno.ENOENT, os.strerror(errno.ENOENT), top
-                    )
-                onerror(exc)
-            return []
-
-        key = self._get_key(top)
-        for prefix, dirs, files in self.trie.walk(key, topdown=topdown):
-            if prefix:
-                root = os.path.join(self.fs_root, os.sep.join(prefix))
-            else:
-                root = self.fs_root
-            if use_dvcignore:
-                dirs[:], files[:] = self.dvcignore(
-                    root, dirs, files, ignore_subrepos=ignore_subrepos,
-                )
-            yield root, dirs, files
-
-    def isexec(self, path_info):
-        if not self.exists(path_info):
-            return False
-
-        mode = self.stat(path_info).st_mode
-        return is_exec(mode)
-
-    def info(self, path_info):
-        key = self._get_key(path_info)
-        try:
-            st = self.trie.stat(key)
-        except KeyError:
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), path_info
-            )
-        return {"size": st.st_size}
-
-    def stat(self, path):
-        key = self._get_key(path)
-        try:
-            return self.trie.stat(key)
-        except KeyError:
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), path
-            )
-
-    def walk_files(self, top):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top):
-            for file in files:
-                # NOTE: os.path.join is ~5.5 times slower
-                yield f"{root}{os.sep}{file}"
-
-    def _reset(self):
-        return self.__dict__.pop("dvcignore", None)

dvc/fs/gs.py

@@ -1,41 +0,0 @@
-import base64
-import threading
-
-from funcy import cached_property, wrap_prop
-
-from dvc.path_info import CloudURLInfo
-from dvc.scheme import Schemes
-
-from .fsspec_wrapper import FSSpecWrapper
-
-
-class GSFileSystem(FSSpecWrapper):
-    scheme = Schemes.GS
-    PATH_CLS = CloudURLInfo
-    REQUIRES = {"gcsfs": "gcsfs"}
-    PARAM_CHECKSUM = "etag"
-    DETAIL_FIELDS = frozenset(("etag", "size"))
-
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-
-        url = config.get("url", "gs://")
-        self.path_info = self.PATH_CLS(url)
-
-    def _prepare_credentials(self, config):
-        login_info = {"consistency": None}
-        login_info["project"] = config.get("projectname")
-        login_info["token"] = config.get("credentialpath")
-        return login_info
-
-    def _entry_hook(self, entry):
-        if "etag" in entry:
-            entry["etag"] = base64.b64decode(entry["etag"]).hex()
-        return entry
-
-    @wrap_prop(threading.Lock())
-    @cached_property
-    def fs(self):
-        from gcsfs import GCSFileSystem
-
-        return GCSFileSystem(**self.fs_args)

dvc/fs/hdfs.py

@@ -1,275 +0,0 @@
-import io
-import logging
-import os
-import re
-import shutil
-import subprocess
-from collections import deque
-from contextlib import closing, contextmanager
-from urllib.parse import urlparse
-
-from dvc.hash_info import HashInfo
-from dvc.progress import Tqdm
-from dvc.scheme import Schemes
-from dvc.utils import fix_env, tmp_fname
-
-from .base import BaseFileSystem, RemoteCmdError
-from .pool import get_connection
-
-logger = logging.getLogger(__name__)
-
-
-def _hadoop_fs(cmd, user=None):
-    cmd = "hadoop fs -" + cmd
-    if user:
-        cmd = f"HADOOP_USER_NAME={user} " + cmd
-
-    # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
-    # See https://github.com/iterative/dvc/issues/1197.
-    close_fds = os.name != "nt"
-
-    executable = os.getenv("SHELL") if os.name != "nt" else None
-    p = subprocess.Popen(
-        cmd,
-        shell=True,
-        close_fds=close_fds,
-        executable=executable,
-        env=fix_env(os.environ),
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-    out, err = p.communicate()
-    if p.returncode != 0:
-        raise RemoteCmdError("hdfs", cmd, p.returncode, err)
-    return out.decode("utf-8")
-
-
-def _group(regex, s, gname):
-    match = re.match(regex, s)
-    assert match is not None
-    return match.group(gname)
-
-
-def _hadoop_fs_checksum(path_info):
-    # NOTE: pyarrow doesn't support checksum, so we need to use hadoop
-    regex = r".*\t.*\t(?P<checksum>.*)"
-    stdout = _hadoop_fs(f"checksum {path_info.url}", user=path_info.user)
-    return _group(regex, stdout, "checksum")
-
-
-class HDFSFileSystem(BaseFileSystem):
-    scheme = Schemes.HDFS
-    REQUIRES = {"pyarrow": "pyarrow"}
-    REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
-    PARAM_CHECKSUM = "checksum"
-    TRAVERSE_PREFIX_LEN = 2
-
-    def __init__(self, repo, config):
-        super().__init__(repo, config)
-
-        self.path_info = None
-        url = config.get("url")
-        if not url:
-            return
-
-        parsed = urlparse(url)
-        user = parsed.username or config.get("user")
-
-        self.path_info = self.PATH_CLS.from_parts(
-            scheme=self.scheme,
-            host=parsed.hostname,
-            user=user,
-            port=parsed.port,
-            path=parsed.path,
-        )
-
-    @staticmethod
-    def hdfs(path_info):
-        import pyarrow.fs
-
-        # NOTE: HadoopFileSystem is not meant to be closed by us and doesn't
-        # have close() method or alternative, so we add a noop one to satisfy
-        # our connection pool.
-        class HDFSConnection(pyarrow.fs.HadoopFileSystem):
-            def close(self):
-                pass
-
-        return get_connection(
-            HDFSConnection,
-            path_info.host,
-            path_info.port,
-            user=path_info.user,
-        )
-
-    @contextmanager
-    def open(self, path_info, mode="r", encoding=None, **kwargs):
-        assert mode in {"r", "rt", "rb"}
-
-        try:
-            with self.hdfs(path_info) as hdfs, closing(
-                hdfs.open_input_stream(path_info.path)
-            ) as fd:
-                if mode == "rb":
-                    yield fd
-                else:
-                    yield io.TextIOWrapper(fd, encoding=encoding)
-        except OSError as e:
-            # Empty .errno and not specific enough error class in pyarrow,
-            # see https://issues.apache.org/jira/browse/ARROW-6248
-            if "file does not exist" in str(e):
-                raise FileNotFoundError(*e.args)
-            raise
-
-    def exists(self, path_info, use_dvcignore=True):
-        assert not isinstance(path_info, list)
-        assert path_info.scheme == "hdfs"
-        with self.hdfs(path_info) as hdfs:
-            import pyarrow.fs
-
-            file_info = hdfs.get_file_info(path_info.path)
-            return file_info.type != pyarrow.fs.FileType.NotFound
-
-    def _walk(self, hdfs, root, topdown=True):
-        import posixpath
-
-        from pyarrow.fs import FileSelector, FileType
-
-        dirs = deque()
-        nondirs = deque()
-
-        selector = FileSelector(root)
-        for entry in hdfs.get_file_info(selector):
-            if entry.type == FileType.Directory:
-                dirs.append(entry.base_name)
-            else:
-                nondirs.append(entry.base_name)
-
-        if topdown:
-            yield root, dirs, nondirs
-
-        for dname in dirs:
-            # NOTE: posixpath.join() is slower
-            yield from self._walk(
-                hdfs, f"{root}{posixpath.sep}{dname}", topdown=topdown
-            )
-
-        if not topdown:
-            yield root, dirs, nondirs
-
-    def walk(self, path_info, **kwargs):
-        if not self.isdir(path_info):
-            return
-
-        with self.hdfs(path_info) as hdfs:
-            for root, dnames, fnames in self._walk(
-                hdfs, path_info.path, **kwargs
-            ):
-                yield path_info.replace(path=root), dnames, fnames
-
-    def walk_files(self, path_info, **kwargs):
-        for root, _, fnames in self.walk(path_info):
-            yield from (root / fname for fname in fnames)
-
-    def remove(self, path_info):
-        if path_info.scheme != "hdfs":
-            raise NotImplementedError
-
-        if self.exists(path_info):
-            logger.debug(f"Removing {path_info.path}")
-            with self.hdfs(path_info) as hdfs:
-                import pyarrow.fs
-
-                file_info = hdfs.get_file_info(path_info.path)
-                if file_info.type == pyarrow.fs.FileType.Directory:
-                    hdfs.delete_dir(path_info.path)
-                else:
-                    hdfs.delete_file(path_info.path)
-
-    def makedirs(self, path_info):
-        with self.hdfs(path_info) as hdfs:
-            # NOTE: fs.create_dir creates parents by default
-            hdfs.create_dir(path_info.path)
-
-    def copy(self, from_info, to_info, **_kwargs):
-        # NOTE: hdfs.copy_file is not supported yet in pyarrow
-        with self.hdfs(to_info) as hdfs:
-            # NOTE: this is how `hadoop fs -cp` works too: it copies through
-            # your local machine.
-            with closing(hdfs.open_input_stream(from_info.path)) as from_fobj:
-                tmp_info = to_info.parent / tmp_fname(to_info.name)
-                try:
-                    with closing(
-                        hdfs.open_output_stream(tmp_info.path)
-                    ) as tmp_fobj:
-                        shutil.copyfileobj(from_fobj, tmp_fobj)
-                    hdfs.move(tmp_info.path, to_info.path)
-                except Exception:
-                    self.remove(tmp_info)
-                    raise
-
-    def isfile(self, path_info):
-        with self.hdfs(path_info) as hdfs:
-            import pyarrow.fs
-
-            file_info = hdfs.get_file_info(path_info.path)
-            return file_info.type == pyarrow.fs.FileType.File
-
-    def isdir(self, path_info):
-        with self.hdfs(path_info) as hdfs:
-            import pyarrow.fs
-
-            file_info = hdfs.get_file_info(path_info.path)
-            return file_info.type == pyarrow.fs.FileType.Directory
-
-    def info(self, path_info):
-        with self.hdfs(path_info) as hdfs:
-            finfo = hdfs.get_file_info(path_info.path)
-            return {"size": finfo.size}
-
-    def checksum(self, path_info):
-        return HashInfo(
-            "checksum",
-            _hadoop_fs_checksum(path_info),
-            size=self.getsize(path_info),
-        )
-
-    def _upload_fobj(self, fobj, to_info):
-        with self.hdfs(to_info) as hdfs:
-            with hdfs.open_output_stream(to_info.path) as fdest:
-                shutil.copyfileobj(fobj, fdest)
-
-    def _upload(
-        self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
-    ):
-        with self.hdfs(to_info) as hdfs:
-            tmp_file = tmp_fname(to_info.path)
-            total = os.path.getsize(from_file)
-            with open(from_file, "rb") as fobj:
-                with Tqdm.wrapattr(
-                    fobj,
-                    "read",
-                    desc=name,
-                    total=total,
-                    disable=no_progress_bar,
-                ) as wrapped:
-                    with hdfs.open_output_stream(tmp_file) as sobj:
-                        sobj.write(wrapped.read())
-            hdfs.move(tmp_file, to_info.path)
-
-    def _download(
-        self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
-    ):
-        with self.hdfs(from_info) as hdfs:
-            file_info = hdfs.get_file_info(from_info.path)
-            total = file_info.size
-            with open(to_file, "wb+") as fobj:
-                with Tqdm.wrapattr(
-                    fobj,
-                    "write",
-                    desc=name,
-                    total=total,
-                    disable=no_progress_bar,
-                ) as wrapped:
-                    with hdfs.open_input_stream(from_info.path) as sobj:
-                        wrapped.write(sobj.read())

dvc/fs/https.py

@@ -1,7 +0,0 @@
-from dvc.scheme import Schemes
-
-from .http import HTTPFileSystem
-
-
-class HTTPSFileSystem(HTTPFileSystem):  # pylint:disable=abstract-method
-    scheme = Schemes.HTTPS

dvc/fs/memory.py

@@ -1,34 +0,0 @@
-from .base import BaseFileSystem
-
-
-class MemoryFileSystem(BaseFileSystem):
-    scheme = "local"
-    PARAM_CHECKSUM = "md5"
-
-    def __init__(self, repo, config):
-        from fsspec.implementations.memory import MemoryFileSystem as MemFS
-
-        super().__init__(repo, config)
-
-        self.fs = MemFS()
-
-    def exists(self, path_info, use_dvcignore=True):
-        return self.fs.exists(path_info.fspath)
-
-    def open(self, path_info, mode="r", encoding=None, **kwargs):
-        return self.fs.open(
-            path_info.fspath, mode=mode, encoding=encoding, **kwargs
-        )
-
-    def info(self, path_info):
-        return self.fs.info(path_info.fspath)
-
-    def stat(self, path_info):
-        import os
-
-        info = self.fs.info(path_info.fspath)
-
-        return os.stat_result((0, 0, 0, 0, 0, 0, info["size"], 0, 0, 0))
-
-    def walk_files(self, path_info, **kwargs):
-        raise NotImplementedError

dvc/fs/webdavs.py

@@ -1,7 +0,0 @@
-from dvc.scheme import Schemes
-
-from .webdav import WebDAVFileSystem
-
-
-class WebDAVSFileSystem(WebDAVFileSystem):  # pylint:disable=abstract-method
-    scheme = Schemes.WEBDAVS

dvc/hash_info.py

@@ -1,9 +1,11 @@
 from collections import OrderedDict
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import Dict, Optional
 
 HASH_DIR_SUFFIX = ".dir"
 
+DirInfo = Dict[str, str]
+
 
 @dataclass
 class HashInfo:
@@ -12,15 +14,13 @@ class HashInfo:
 
     name: Optional[str]
     value: Optional[str]
+    dir_info: Optional[DirInfo] = field(default=None, compare=False)
     size: Optional[int] = field(default=None, compare=False)
     nfiles: Optional[int] = field(default=None, compare=False)
 
     def __bool__(self):
         return bool(self.value)
 
-    def __str__(self):
-        return f"{self.name}: {self.value}"
-
     @classmethod
     def from_dict(cls, d):
         _d = d.copy() if d else {}

dvc/ignore.py

@@ -49,11 +49,13 @@ class DvcIgnorePatterns(DvcIgnore):
         ]
 
     @classmethod
-    def from_files(cls, ignore_file_path, fs):
+    def from_files(cls, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
         dirname = os.path.normpath(os.path.dirname(ignore_file_path))
-        ignore_file_rel_path = os.path.relpath(ignore_file_path, fs.fs_root)
-        with fs.open(ignore_file_path, encoding="utf-8") as fobj:
+        ignore_file_rel_path = os.path.relpath(
+            ignore_file_path, tree.tree_root
+        )
+        with tree.open(ignore_file_path, encoding="utf-8") as fobj:
             path_spec_lines = [
                 PatternInfo(
                     line,
@@ -89,50 +91,45 @@ class DvcIgnorePatterns(DvcIgnore):
             path = normalize_file(path)
         return path
 
-    def matches(self, dirname, basename, is_dir=False, details: bool = False):
+    def matches(self, dirname, basename, is_dir=False):
         path = self._get_normalize_path(dirname, basename)
         if not path:
             return False
-
-        if details:
-            return self._ignore_details(path, is_dir)
         return self.ignore(path, is_dir)
 
     def ignore(self, path, is_dir):
-        def matches(pattern, path, is_dir) -> bool:
-            matches_ = bool(pattern.match(path))
-
-            if is_dir:
-                matches_ |= bool(pattern.match(f"{path}/"))
-
-            return matches_
-
         result = False
-
-        for ignore, pattern in self.ignore_spec[::-1]:
-            if matches(pattern, path, is_dir):
-                result = ignore
-                break
+        if is_dir:
+            path_dir = f"{path}/"
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path) or pattern.match(path_dir):
+                    result = ignore
+        else:
+            for ignore, pattern in self.ignore_spec:
+                if pattern.match(path):
+                    result = ignore
         return result
 
-    def _ignore_details(self, path, is_dir: bool):
+    def match_details(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
+        return self._ignore_details(path, is_dir)
+
+    def _ignore_details(self, path, is_dir):
         result = []
-        for (regex, _), pattern_info in list(
-            zip(self.regex_pattern_list, self.pattern_list)
-        ):
+        for ignore, pattern in zip(self.regex_pattern_list, self.pattern_list):
+            regex = re.compile(ignore[0])
             # skip system pattern
-            if not pattern_info.file_info:
+            if not pattern.file_info:
                 continue
-
-            regex = re.compile(regex)
-
-            matches = bool(regex.match(path))
             if is_dir:
-                matches |= bool(regex.match(f"{path}/"))
-
-            if matches:
-                result.append(pattern_info.file_info)
-
+                path_dir = f"{path}/"
+                if regex.match(path) or regex.match(path_dir):
+                    result.append(pattern.file_info)
+            else:
+                if regex.match(path):
+                    result.append(pattern.file_info)
         return result
 
     def __hash__(self):
@@ -160,7 +157,7 @@ def _no_match(path):
 
 
 class DvcIgnoreFilterNoop:
-    def __init__(self, fs, root_dir):
+    def __init__(self, tree, root_dir):
         pass
 
     def __call__(self, root, dirs, files, **kwargs):
@@ -180,35 +177,39 @@ class DvcIgnoreFilterNoop:
 
 
 class DvcIgnoreFilter:
-    def __init__(self, fs, root_dir):
+    @staticmethod
+    def _is_dvc_repo(root, directory):
+        from dvc.repo import Repo
+
+        return os.path.isdir(os.path.join(root, directory, Repo.DVC_DIR))
+
+    def __init__(self, tree, root_dir):
         from dvc.repo import Repo
 
         default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
 
-        self.fs = fs
+        self.tree = tree
         self.root_dir = root_dir
-        self.ignores_trie_fs = PathStringTrie()
-        self.ignores_trie_fs[root_dir] = DvcIgnorePatterns(
+        self.ignores_trie_tree = PathStringTrie()
+        self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
             default_ignore_patterns, root_dir
         )
         self._ignored_subrepos = PathStringTrie()
         self._update(self.root_dir)
 
     def _update(self, dirname):
-        self._update_sub_repo(dirname)
-
-        old_pattern = self.ignores_trie_fs.longest_prefix(dirname).value
+        old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
         matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
 
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if not matches and self.fs.exists(
+        if not matches and self.tree.exists(
             ignore_file_path, use_dvcignore=False
         ):
             new_pattern = DvcIgnorePatterns.from_files(
-                ignore_file_path, self.fs
+                ignore_file_path, self.tree
             )
             if old_pattern:
-                self.ignores_trie_fs[dirname] = DvcIgnorePatterns(
+                self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
                     *merge_patterns(
                         old_pattern.pattern_list,
                         old_pattern.dirname,
@@ -217,43 +218,39 @@ class DvcIgnoreFilter:
                     )
                 )
             else:
-                self.ignores_trie_fs[dirname] = new_pattern
+                self.ignores_trie_tree[dirname] = new_pattern
         elif old_pattern:
-            self.ignores_trie_fs[dirname] = old_pattern
+            self.ignores_trie_tree[dirname] = old_pattern
 
-    def _update_sub_repo(self, path):
-        from dvc.repo import Repo
+        # NOTE: using `walk` + `break` because tree doesn't have `listdir()`
+        for root, dirs, _ in self.tree.walk(dirname, use_dvcignore=False):
+            self._update_sub_repo(root, dirs)
+            break
 
-        if path == self.root_dir:
-            return
-
-        dvc_dir = os.path.join(path, Repo.DVC_DIR)
-        if not os.path.exists(dvc_dir):
-            return
-
-        root, dname = os.path.split(path)
-        self._ignored_subrepos[root] = self._ignored_subrepos.get(
-            root, set()
-        ) | {dname}
-        pattern_info = PatternInfo(f"/{dname}/", f"in sub_repo:{dname}")
-        new_pattern = DvcIgnorePatterns([pattern_info], root)
-        old_pattern = self.ignores_trie_fs.longest_prefix(root).value
-        if old_pattern:
-            self.ignores_trie_fs[root] = DvcIgnorePatterns(
-                *merge_patterns(
-                    old_pattern.pattern_list,
-                    old_pattern.dirname,
-                    new_pattern.pattern_list,
-                    new_pattern.dirname,
+    def _update_sub_repo(self, root, dirs):
+        for d in dirs:
+            if self._is_dvc_repo(root, d):
+                self._ignored_subrepos[root] = self._ignored_subrepos.get(
+                    root, set()
+                ) | {d}
+                pattern_info = PatternInfo(
+                    f"/{d}/", "in sub_repo:{}".format(d)
                 )
-            )
-        else:
-            self.ignores_trie_fs[root] = new_pattern
+                new_pattern = DvcIgnorePatterns([pattern_info], root)
+                old_pattern = self.ignores_trie_tree.longest_prefix(root).value
+                if old_pattern:
+                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
+                        *merge_patterns(
+                            old_pattern.pattern_list,
+                            old_pattern.dirname,
+                            new_pattern.pattern_list,
+                            new_pattern.dirname,
+                        )
+                    )
+                else:
+                    self.ignores_trie_tree[root] = new_pattern
 
     def __call__(self, root, dirs, files, ignore_subrepos=True):
-        for dname in dirs:
-            self._update_sub_repo(os.path.join(root, dname))
-
         ignore_pattern = self._get_trie_pattern(root)
         if ignore_pattern:
             dirs, files = ignore_pattern(root, dirs, files)
@@ -262,11 +259,11 @@ class DvcIgnoreFilter:
         return dirs, files
 
     def _get_trie_pattern(self, dirname):
-        ignore_pattern = self.ignores_trie_fs.get(dirname)
+        ignore_pattern = self.ignores_trie_tree.get(dirname)
         if ignore_pattern:
             return ignore_pattern
 
-        prefix = self.ignores_trie_fs.longest_prefix(dirname).key
+        prefix = self.ignores_trie_tree.longest_prefix(dirname).key
         if not prefix:
             # outside of the repo
             return None
@@ -283,7 +280,7 @@ class DvcIgnoreFilter:
         for parent in dirs:
             self._update(parent)
 
-        return self.ignores_trie_fs.get(dirname)
+        return self.ignores_trie_tree.get(dirname)
 
     def _is_ignored(self, path, is_dir=False):
         if self._outside_repo(path):
@@ -292,7 +289,8 @@ class DvcIgnoreFilter:
         ignore_pattern = self._get_trie_pattern(dirname)
         if ignore_pattern:
             return ignore_pattern.matches(dirname, basename, is_dir)
-        return False
+        else:
+            return False
 
     def _is_subrepo(self, path):
         dirname, basename = os.path.split(os.path.normpath(path))
@@ -326,15 +324,13 @@ class DvcIgnoreFilter:
         return False
 
     def check_ignore(self, target):
-        # NOTE: can only be used in `dvc check-ignore`, see
-        # https://github.com/iterative/dvc/issues/5046
         full_target = os.path.abspath(target)
         if not self._outside_repo(full_target):
             dirname, basename = os.path.split(os.path.normpath(full_target))
             pattern = self._get_trie_pattern(dirname)
             if pattern:
-                matches = pattern.matches(
-                    dirname, basename, os.path.isdir(full_target), True,
+                matches = pattern.match_details(
+                    dirname, basename, os.path.isdir(full_target)
                 )
 
                 if matches:
@@ -344,11 +340,7 @@ class DvcIgnoreFilter:
     def is_ignored(self, path):
         # NOTE: can't use self.check_ignore(path).match for now, see
         # https://github.com/iterative/dvc/issues/4555
-        if os.path.isfile(path):
-            return self.is_ignored_file(path)
-        if os.path.isdir(path):
-            return self.is_ignored_dir(path)
-        return self.is_ignored_file(path) or self.is_ignored_dir(path)
+        return self.is_ignored_dir(path) or self.is_ignored_file(path)
 
 
 def init(path):

dvc/info.py

@@ -4,17 +4,20 @@ import pathlib
 import platform
 import uuid
 
-import psutil
-
 from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.fs import FS_MAP, get_fs_cls, get_fs_config
 from dvc.repo import Repo
 from dvc.scm.base import SCMError
 from dvc.system import System
+from dvc.tree import TREES, get_tree_cls, get_tree_config
 from dvc.utils import error_link
 from dvc.utils.pkg import PKG
 from dvc.version import __version__
 
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
 if PKG is None:
     package = ""
 else:
@@ -37,16 +40,17 @@ def get_dvc_info():
         # can't auto-create it, as it might cause issues if the user
         # later decides to enable shared cache mode with
         # `dvc config cache.shared group`.
-        if os.path.exists(repo.odb.local.cache_dir):
+        if os.path.exists(repo.cache.local.cache_dir):
             info.append(
                 "Cache types: {}".format(_get_linktype_support_info(repo))
             )
-            fs_type = get_fs_type(repo.odb.local.cache_dir)
-            info.append(f"Cache directory: {fs_type}")
+            if psutil:
+                fs_type = get_fs_type(repo.cache.local.cache_dir)
+                info.append(f"Cache directory: {fs_type}")
         else:
             info.append("Cache types: " + error_link("no-dvc-cache"))
 
-        info.append(f"Caches: {_get_caches(repo.odb)}")
+        info.append(f"Caches: {_get_caches(repo.cache)}")
 
         info.append(f"Remotes: {_get_remotes(repo.config)}")
 
@@ -56,8 +60,9 @@ def get_dvc_info():
         info.append("Repo: dvc, git (broken)")
     else:
         root_directory = repo.root_dir
-        fs_root = get_fs_type(os.path.abspath(root_directory))
-        info.append(f"Workspace directory: {fs_root}")
+        if psutil:
+            fs_root = get_fs_type(os.path.abspath(root_directory))
+            info.append(f"Workspace directory: {fs_root}")
         info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
     return "\n".join(info)
 
@@ -75,7 +80,7 @@ def _get_caches(cache):
 
 def _get_remotes(config):
     schemes = (
-        get_fs_cls(get_fs_config(config, name=remote)).scheme
+        get_tree_cls(get_tree_config(config, name=remote)).scheme
         for remote in config["remote"]
     )
 
@@ -91,7 +96,7 @@ def _get_linktype_support_info(repo):
     }
 
     fname = "." + str(uuid.uuid4())
-    src = os.path.join(repo.odb.local.cache_dir, fname)
+    src = os.path.join(repo.cache.local.cache_dir, fname)
     open(src, "w").close()
     dst = os.path.join(repo.root_dir, fname)
 
@@ -117,11 +122,11 @@ def _get_linktype_support_info(repo):
 def _get_supported_remotes():
 
     supported_remotes = []
-    for scheme, fs_cls in FS_MAP.items():
-        if not fs_cls.get_missing_deps():
-            supported_remotes.append(scheme)
+    for tree_cls in TREES:
+        if not tree_cls.get_missing_deps():
+            supported_remotes.append(tree_cls.scheme)
 
-    if len(supported_remotes) == len(FS_MAP):
+    if len(supported_remotes) == len(TREES):
         return "All remotes"
 
     if len(supported_remotes) == 1:
@@ -137,8 +142,7 @@ def get_fs_type(path):
         for part in psutil.disk_partitions(all=True)
     }
 
-    # need to follow the symlink: https://github.com/iterative/dvc/issues/5065
-    path = pathlib.Path(path).resolve()
+    path = pathlib.Path(path)
 
     for parent in itertools.chain([path], path.parents):
         if parent in partition:

dvc/istextfile.py

@@ -7,7 +7,19 @@
 TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
 
 
-def istextblock(block):
+def istextfile(fname, blocksize=512, tree=None):
+    """ Uses heuristics to guess whether the given file is text or binary,
+        by reading a single block of bytes from the file.
+        If more than 30% of the chars in the block are non-text, or there
+        are NUL ('\x00') bytes in the block, assume this is a binary file.
+    """
+    if tree:
+        open_func = tree.open
+    else:
+        open_func = open
+    with open_func(fname, "rb") as fobj:
+        block = fobj.read(blocksize)
+
     if not block:
         # An empty file is considered a valid text file
         return True
@@ -20,14 +32,3 @@ def istextblock(block):
     # occurrences of TEXT_CHARS from the block
     nontext = block.translate(None, TEXT_CHARS)
     return float(len(nontext)) / len(block) <= 0.30
-
-
-def istextfile(fname, fs, blocksize=512):
-    """ Uses heuristics to guess whether the given file is text or binary,
-        by reading a single block of bytes from the file.
-        If more than 30% of the chars in the block are non-text, or there
-        are NUL ('\x00') bytes in the block, assume this is a binary file.
-    """
-    with fs.open(fname, "rb") as fobj:
-        block = fobj.read(blocksize)
-    return istextblock(block)

dvc/lock.py

@@ -148,9 +148,10 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
     ):  # pylint: disable=super-init-not-called
         import socket
 
-        self._tmp_dir = tmp_dir
         super().__init__(lockfile)
 
+        self._tmp_dir = tmp_dir
+
         # NOTE: this is basically Lock.__init__ copy-paste, except that
         # instead of using `socket.getfqdn()` we use `socket.gethostname()`
         # to speed this up. We've seen [1] `getfqdn()` take ~5sec to return

dvc/logger.py

@@ -23,19 +23,14 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     Adds a new logging level to the `logging` module and the
     currently configured logging class.
 
-    Uses the existing numeric levelNum if already defined.
-
     Based on https://stackoverflow.com/questions/2183233
     """
     if methodName is None:
         methodName = levelName.lower()
 
-    # If the level name is already defined as a top-level `logging`
-    # constant, then adopt the existing numeric level.
-    if hasattr(logging, levelName):
-        existingLevelNum = getattr(logging, levelName)
-        assert isinstance(existingLevelNum, int)
-        levelNum = existingLevelNum
+    assert not hasattr(logging, levelName)
+    assert not hasattr(logging, methodName)
+    assert not hasattr(logging.getLoggerClass(), methodName)
 
     def logForLevel(self, message, *args, **kwargs):
         if self.isEnabledFor(levelNum):
@@ -45,19 +40,10 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     def logToRoot(message, *args, **kwargs):
         logging.log(levelNum, message, *args, **kwargs)
 
-    # getLevelName resolves the numeric log level if already defined,
-    # otherwise returns a string
-    if not isinstance(logging.getLevelName(levelName), int):
-        logging.addLevelName(levelNum, levelName)
-
-    if not hasattr(logging, levelName):
-        setattr(logging, levelName, levelNum)
-
-    if not hasattr(logging.getLoggerClass(), methodName):
-        setattr(logging.getLoggerClass(), methodName, logForLevel)
-
-    if not hasattr(logging, methodName):
-        setattr(logging, methodName, logToRoot)
+    logging.addLevelName(levelNum, levelName)
+    setattr(logging, levelName, levelNum)
+    setattr(logging.getLoggerClass(), methodName, logForLevel)
+    setattr(logging, methodName, logToRoot)
 
 
 class LoggingException(Exception):

dvc/main.py

@@ -1,11 +1,17 @@
 """Main entry point for dvc CLI."""
-import logging
 
-from dvc._debug import debugtools
+import errno
+import logging
+from contextlib import contextmanager
+
+from dvc import analytics
 from dvc.cli import parse_args
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
+from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER, disable_other_loggers
+from dvc.tree.pool import close_pools
+from dvc.utils import error_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
@@ -16,6 +22,39 @@ from dvc.logger import FOOTER, disable_other_loggers
 logger = logging.getLogger("dvc")
 
 
+@contextmanager
+def profile(enable, dump):
+    if not enable:
+        yield
+        return
+
+    import cProfile
+
+    prof = cProfile.Profile()
+    prof.enable()
+
+    yield
+
+    prof.disable()
+    if not dump:
+        prof.print_stats(sort="cumtime")
+        return
+    prof.dump_stats(dump)
+
+
+@contextmanager
+def debug(enable):
+    try:
+        yield
+        return
+    except Exception:
+        if enable:
+            import pdb  # noqa: T100
+
+            pdb.post_mortem()
+        raise
+
+
 def main(argv=None):  # noqa: C901
     """Run dvc CLI command.
 
@@ -45,14 +84,10 @@ def main(argv=None):  # noqa: C901
 
         logger.trace(args)
 
-        if not args.quiet:
-            from dvc.ui import ui
-
-            ui.enable()
-
-        with debugtools(args):
-            cmd = args.func(args)
-            ret = cmd.run()
+        with profile(enable=args.cprofile, dump=args.cprofile_dump):
+            with debug(args.pdb):
+                cmd = args.func(args)
+                ret = cmd.run()
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -69,11 +104,7 @@ def main(argv=None):  # noqa: C901
         logger.exception("")
     except Exception as exc:  # noqa, pylint: disable=broad-except
         # pylint: disable=no-member
-        import errno
-
         if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
-            from dvc.utils import error_link
-
             logger.exception(
                 "too many open files, please visit "
                 "{} to see how to handle this "
@@ -92,8 +123,6 @@ def main(argv=None):  # noqa: C901
         ret = 255
 
     try:
-        from dvc import analytics
-
         if analytics.is_enabled():
             analytics.collect_and_send_report(args, ret)
 
@@ -101,15 +130,11 @@ def main(argv=None):  # noqa: C901
     finally:
         logger.setLevel(outerLogLevel)
 
-        from dvc.fs.pool import close_pools
-
         # Closing pools by-hand to prevent weird messages when closing SSH
         # connections. See https://github.com/iterative/dvc/issues/3248 for
         # more info.
         close_pools()
 
-        from dvc.external_repo import clean_repos
-
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()

dvc/objects/__init__.py

@@ -1,28 +0,0 @@
-import logging
-
-from dvc.progress import Tqdm
-
-from .tree import Tree
-
-logger = logging.getLogger(__name__)
-
-
-def save(odb, obj, **kwargs):
-    if isinstance(obj, Tree):
-        for _, entry in Tqdm(obj):
-            odb.add(entry.path_info, entry.fs, entry.hash_info, **kwargs)
-    odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)
-
-
-def check(odb, obj):
-    odb.check(obj.hash_info)
-
-    if isinstance(obj, Tree):
-        for _, entry in obj:
-            odb.check(entry.hash_info)
-
-
-def load(odb, hash_info):
-    if hash_info.isdir:
-        return Tree.load(odb, hash_info)
-    return odb.get(hash_info)

dvc/objects/db/base.py

@@ -1,412 +0,0 @@
-import itertools
-import logging
-from concurrent.futures import ThreadPoolExecutor
-from copy import copy
-from typing import Optional
-
-from dvc.objects.errors import ObjectFormatError
-from dvc.objects.file import HashFile
-from dvc.progress import Tqdm
-
-logger = logging.getLogger(__name__)
-
-
-class ObjectDB:
-
-    DEFAULT_VERIFY = False
-    DEFAULT_CACHE_TYPES = ["copy"]
-    CACHE_MODE: Optional[int] = None
-
-    def __init__(self, fs):
-        self.fs = fs
-        self.repo = fs.repo
-
-        self.verify = fs.config.get("verify", self.DEFAULT_VERIFY)
-        self.cache_types = fs.config.get("type") or copy(
-            self.DEFAULT_CACHE_TYPES
-        )
-        self.cache_type_confirmed = False
-
-    def move(self, from_info, to_info):
-        self.fs.move(from_info, to_info)
-
-    def makedirs(self, path_info):
-        self.fs.makedirs(path_info)
-
-    def get(self, hash_info):
-        """ get raw object """
-        return HashFile(
-            # Prefer string path over PathInfo when possible due to performance
-            self.hash_to_path(hash_info.value),
-            self.fs,
-            hash_info,
-        )
-
-    def add(self, path_info, fs, hash_info, **kwargs):
-        try:
-            self.check(hash_info)
-            return
-        except (ObjectFormatError, FileNotFoundError):
-            pass
-
-        cache_info = self.hash_to_path_info(hash_info.value)
-        # using our makedirs to create dirs with proper permissions
-        self.makedirs(cache_info.parent)
-        if isinstance(fs, type(self.fs)):
-            self.fs.move(path_info, cache_info)
-        else:
-            with fs.open(path_info, mode="rb") as fobj:
-                self.fs.upload_fobj(fobj, cache_info)
-        self.protect(cache_info)
-        self.fs.repo.state.save(cache_info, self.fs, hash_info)
-
-        callback = kwargs.get("download_callback")
-        if callback:
-            callback(1)
-
-    def hash_to_path_info(self, hash_):
-        return self.fs.path_info / hash_[0:2] / hash_[2:]
-
-    # Override to return path as a string instead of PathInfo for clouds
-    # which support string paths (see local)
-    def hash_to_path(self, hash_):
-        return self.hash_to_path_info(hash_)
-
-    def protect(self, path_info):  # pylint: disable=unused-argument
-        pass
-
-    def is_protected(self, path_info):  # pylint: disable=unused-argument
-        return False
-
-    def unprotect(self, path_info):  # pylint: disable=unused-argument
-        pass
-
-    def set_exec(self, path_info):  # pylint: disable=unused-argument
-        pass
-
-    def check(self, hash_info):
-        """Compare the given hash with the (corresponding) actual one.
-
-        - Use `State` as a cache for computed hashes
-            + The entries are invalidated by taking into account the following:
-                * mtime
-                * inode
-                * size
-                * hash
-
-        - Remove the file from cache if it doesn't match the actual hash
-        """
-
-        obj = self.get(hash_info)
-        if self.is_protected(obj.path_info):
-            logger.trace(
-                "Assuming '%s' is unchanged since it is read-only",
-                obj.path_info,
-            )
-            return
-
-        try:
-            obj.check(self)
-        except ObjectFormatError:
-            logger.warning("corrupted cache file '%s'.", obj.path_info)
-            self.fs.remove(obj.path_info)
-            raise
-
-        # making cache file read-only so we don't need to check it
-        # next time
-        self.protect(obj.path_info)
-
-    def _list_paths(self, prefix=None, progress_callback=None):
-        if prefix:
-            if len(prefix) > 2:
-                path_info = self.fs.path_info / prefix[:2] / prefix[2:]
-            else:
-                path_info = self.fs.path_info / prefix[:2]
-            prefix = True
-        else:
-            path_info = self.fs.path_info
-            prefix = False
-        if progress_callback:
-            for file_info in self.fs.walk_files(path_info, prefix=prefix):
-                progress_callback()
-                yield file_info.path
-        else:
-            yield from self.fs.walk_files(path_info, prefix=prefix)
-
-    def _path_to_hash(self, path):
-        parts = self.fs.PATH_CLS(path).parts[-2:]
-
-        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
-            raise ValueError(f"Bad cache file path '{path}'")
-
-        return "".join(parts)
-
-    def list_hashes(self, prefix=None, progress_callback=None):
-        """Iterate over hashes in this fs.
-
-        If `prefix` is specified, only hashes which begin with `prefix`
-        will be returned.
-        """
-        for path in self._list_paths(prefix, progress_callback):
-            try:
-                yield self._path_to_hash(path)
-            except ValueError:
-                logger.debug(
-                    "'%s' doesn't look like a cache file, skipping", path
-                )
-
-    def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
-        count = 0
-        for hash_ in self.list_hashes(prefix, progress_callback):
-            yield hash_
-            count += 1
-            if count > limit:
-                logger.debug(
-                    "`list_hashes()` returned max '{}' hashes, "
-                    "skipping remaining results".format(limit)
-                )
-                return
-
-    def _max_estimation_size(self, hashes):
-        # Max remote size allowed for us to use traverse method
-        return max(
-            self.fs.TRAVERSE_THRESHOLD_SIZE,
-            len(hashes)
-            / self.fs.TRAVERSE_WEIGHT_MULTIPLIER
-            * self.fs.LIST_OBJECT_PAGE_SIZE,
-        )
-
-    def _estimate_remote_size(self, hashes=None, name=None):
-        """Estimate fs size based on number of entries beginning with
-        "00..." prefix.
-        """
-        prefix = "0" * self.fs.TRAVERSE_PREFIX_LEN
-        total_prefixes = pow(16, self.fs.TRAVERSE_PREFIX_LEN)
-        if hashes:
-            max_hashes = self._max_estimation_size(hashes)
-        else:
-            max_hashes = None
-
-        with Tqdm(
-            desc="Estimating size of "
-            + (f"cache in '{name}'" if name else "remote cache"),
-            unit="file",
-        ) as pbar:
-
-            def update(n=1):
-                pbar.update(n * total_prefixes)
-
-            if max_hashes:
-                hashes = self._hashes_with_limit(
-                    max_hashes / total_prefixes, prefix, update
-                )
-            else:
-                hashes = self.list_hashes(prefix, update)
-
-            remote_hashes = set(hashes)
-            if remote_hashes:
-                remote_size = total_prefixes * len(remote_hashes)
-            else:
-                remote_size = total_prefixes
-            logger.debug(f"Estimated remote size: {remote_size} files")
-        return remote_size, remote_hashes
-
-    def list_hashes_traverse(
-        self, remote_size, remote_hashes, jobs=None, name=None
-    ):
-        """Iterate over all hashes found in this fs.
-        Hashes are fetched in parallel according to prefix, except in
-        cases where the remote size is very small.
-
-        All hashes from the remote (including any from the size
-        estimation step passed via the `remote_hashes` argument) will be
-        returned.
-
-        NOTE: For large remotes the list of hashes will be very
-        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
-        and we don't really need all of it at the same time, so it makes
-        sense to use a generator to gradually iterate over it, without
-        keeping all of it in memory.
-        """
-        num_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
-        if num_pages < 256 / self.fs.JOBS:
-            # Fetching prefixes in parallel requires at least 255 more
-            # requests, for small enough remotes it will be faster to fetch
-            # entire cache without splitting it into prefixes.
-            #
-            # NOTE: this ends up re-fetching hashes that were already
-            # fetched during remote size estimation
-            traverse_prefixes = [None]
-            initial = 0
-        else:
-            yield from remote_hashes
-            initial = len(remote_hashes)
-            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
-            if self.fs.TRAVERSE_PREFIX_LEN > 2:
-                traverse_prefixes += [
-                    "{0:0{1}x}".format(i, self.fs.TRAVERSE_PREFIX_LEN)
-                    for i in range(1, pow(16, self.fs.TRAVERSE_PREFIX_LEN - 2))
-                ]
-        with Tqdm(
-            desc="Querying "
-            + (f"cache in '{name}'" if name else "remote cache"),
-            total=remote_size,
-            initial=initial,
-            unit="file",
-        ) as pbar:
-
-            def list_with_update(prefix):
-                return list(
-                    self.list_hashes(
-                        prefix=prefix, progress_callback=pbar.update
-                    )
-                )
-
-            with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
-            ) as executor:
-                in_remote = executor.map(list_with_update, traverse_prefixes,)
-                yield from itertools.chain.from_iterable(in_remote)
-
-    def all(self, jobs=None, name=None):
-        """Iterate over all hashes in this fs.
-
-        Hashes will be fetched in parallel threads according to prefix
-        (except for small remotes) and a progress bar will be displayed.
-        """
-        logger.debug(
-            "Fetching all hashes from '{}'".format(
-                name if name else "remote cache"
-            )
-        )
-
-        if not self.fs.CAN_TRAVERSE:
-            return self.list_hashes()
-
-        remote_size, remote_hashes = self._estimate_remote_size(name=name)
-        return self.list_hashes_traverse(
-            remote_size, remote_hashes, jobs, name
-        )
-
-    def _remove_unpacked_dir(self, hash_):
-        pass
-
-    def gc(self, used, jobs=None):
-        removed = False
-        # hashes must be sorted to ensure we always remove .dir files first
-        for hash_ in sorted(
-            self.all(jobs, str(self.fs.path_info)),
-            key=self.fs.is_dir_hash,
-            reverse=True,
-        ):
-            if hash_ in used:
-                continue
-            path_info = self.hash_to_path_info(hash_)
-            if self.fs.is_dir_hash(hash_):
-                # backward compatibility
-                # pylint: disable=protected-access
-                self._remove_unpacked_dir(hash_)
-            self.fs.remove(path_info)
-            removed = True
-
-        return removed
-
-    def list_hashes_exists(self, hashes, jobs=None, name=None):
-        """Return list of the specified hashes which exist in this fs.
-        Hashes will be queried individually.
-        """
-        logger.debug(
-            "Querying {} hashes via object_exists".format(len(hashes))
-        )
-        with Tqdm(
-            desc="Querying "
-            + ("cache in " + name if name else "remote cache"),
-            total=len(hashes),
-            unit="file",
-        ) as pbar:
-
-            def exists_with_progress(path_info):
-                ret = self.fs.exists(path_info)
-                pbar.update_msg(str(path_info))
-                return ret
-
-            with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
-            ) as executor:
-                path_infos = map(self.hash_to_path_info, hashes)
-                in_remote = executor.map(exists_with_progress, path_infos)
-                ret = list(itertools.compress(hashes, in_remote))
-                return ret
-
-    def hashes_exist(self, hashes, jobs=None, name=None):
-        """Check if the given hashes are stored in the remote.
-
-        There are two ways of performing this check:
-
-        - Traverse method: Get a list of all the files in the remote
-            (traversing the cache directory) and compare it with
-            the given hashes. Cache entries will be retrieved in parallel
-            threads according to prefix (i.e. entries starting with, "00...",
-            "01...", and so on) and a progress bar will be displayed.
-
-        - Exists method: For each given hash, run the `exists`
-            method and filter the hashes that aren't on the remote.
-            This is done in parallel threads.
-            It also shows a progress bar when performing the check.
-
-        The reason for such an odd logic is that most of the remotes
-        take much shorter time to just retrieve everything they have under
-        a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
-        check if particular file exists much quicker, use their own
-        implementation of hashes_exist (see ssh, local).
-
-        Which method to use will be automatically determined after estimating
-        the size of the remote cache, and comparing the estimated size with
-        len(hashes). To estimate the size of the remote cache, we fetch
-        a small subset of cache entries (i.e. entries starting with "00...").
-        Based on the number of entries in that subset, the size of the full
-        cache can be estimated, since the cache is evenly distributed according
-        to hash.
-
-        Returns:
-            A list with hashes that were found in the remote
-        """
-        # Remotes which do not use traverse prefix should override
-        # hashes_exist() (see ssh, local)
-        assert self.fs.TRAVERSE_PREFIX_LEN >= 2
-
-        hashes = set(hashes)
-        if len(hashes) == 1 or not self.fs.CAN_TRAVERSE:
-            remote_hashes = self.list_hashes_exists(hashes, jobs, name)
-            return remote_hashes
-
-        # Max remote size allowed for us to use traverse method
-        remote_size, remote_hashes = self._estimate_remote_size(hashes, name)
-
-        traverse_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
-        # For sufficiently large remotes, traverse must be weighted to account
-        # for performance overhead from large lists/sets.
-        # From testing with S3, for remotes with 1M+ files, object_exists is
-        # faster until len(hashes) is at least 10k~100k
-        if remote_size > self.fs.TRAVERSE_THRESHOLD_SIZE:
-            traverse_weight = (
-                traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER
-            )
-        else:
-            traverse_weight = traverse_pages
-        if len(hashes) < traverse_weight:
-            logger.debug(
-                "Large remote ('{}' hashes < '{}' traverse weight), "
-                "using object_exists for remaining hashes".format(
-                    len(hashes), traverse_weight
-                )
-            )
-            return list(hashes & remote_hashes) + self.list_hashes_exists(
-                hashes - remote_hashes, jobs, name
-            )
-
-        logger.debug("Querying '{}' hashes via traverse".format(len(hashes)))
-        remote_hashes = set(
-            self.list_hashes_traverse(remote_size, remote_hashes, jobs, name)
-        )
-        return list(hashes & set(remote_hashes))

dvc/objects/db/gdrive.py

@@ -1,5 +0,0 @@
-from .base import ObjectDB
-
-
-class GDriveObjectDB(ObjectDB):
-    DEFAULT_VERIFY = True

dvc/objects/db/local.py

@@ -1,172 +0,0 @@
-import logging
-import os
-import stat
-
-from funcy import cached_property
-from shortuuid import uuid
-
-from dvc.hash_info import HashInfo
-from dvc.objects.errors import ObjectFormatError
-from dvc.path_info import PathInfo
-from dvc.progress import Tqdm
-from dvc.utils import relpath
-from dvc.utils.fs import copyfile, remove, umask, walk_files
-
-from .base import ObjectDB
-
-logger = logging.getLogger(__name__)
-
-
-class LocalObjectDB(ObjectDB):
-    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-    CACHE_MODE = 0o444
-    UNPACKED_DIR_SUFFIX = ".unpacked"
-
-    def __init__(self, fs):
-        super().__init__(fs)
-        self.cache_dir = fs.config.get("url")
-
-        shared = fs.config.get("shared")
-        if shared:
-            self._file_mode = 0o664
-            self._dir_mode = 0o2775
-        else:
-            self._file_mode = 0o666 & ~umask
-            self._dir_mode = 0o777 & ~umask
-
-    @property
-    def cache_dir(self):
-        return self.fs.path_info.fspath if self.fs.path_info else None
-
-    @cache_dir.setter
-    def cache_dir(self, value):
-        self.fs.path_info = PathInfo(value) if value else None
-
-    @cached_property
-    def cache_path(self):
-        return os.path.abspath(self.cache_dir)
-
-    def move(self, from_info, to_info):
-        super().move(from_info, to_info)
-        os.chmod(to_info, self._file_mode)
-
-    def makedirs(self, path_info):
-        from dvc.utils.fs import makedirs
-
-        makedirs(path_info, exist_ok=True, mode=self._dir_mode)
-
-    def hash_to_path(self, hash_):
-        # NOTE: `self.cache_path` is already normalized so we can simply use
-        # `os.sep` instead of `os.path.join`. This results in this helper
-        # being ~5.5 times faster.
-        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
-
-    def hashes_exist(
-        self, hashes, jobs=None, name=None
-    ):  # pylint: disable=unused-argument
-        ret = []
-
-        for hash_ in Tqdm(
-            hashes,
-            unit="file",
-            desc="Querying " + ("cache in " + name if name else "local cache"),
-        ):
-            hash_info = HashInfo(self.fs.PARAM_CHECKSUM, hash_)
-            try:
-                self.check(hash_info)
-                ret.append(hash_)
-            except (FileNotFoundError, ObjectFormatError):
-                pass
-
-        return ret
-
-    def _list_paths(self, prefix=None, progress_callback=None):
-        assert self.fs.path_info is not None
-        if prefix:
-            path_info = self.fs.path_info / prefix[:2]
-            if not self.fs.exists(path_info):
-                return
-        else:
-            path_info = self.fs.path_info
-        # NOTE: use utils.fs walk_files since fs.walk_files will not follow
-        # symlinks
-        if progress_callback:
-            for path in walk_files(path_info):
-                progress_callback()
-                yield path
-        else:
-            yield from walk_files(path_info)
-
-    def _remove_unpacked_dir(self, hash_):
-        info = self.hash_to_path_info(hash_)
-        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
-        self.fs.remove(path_info)
-
-    def _unprotect_file(self, path):
-        if self.fs.is_symlink(path) or self.fs.is_hardlink(path):
-            logger.debug("Unprotecting '%s'", path)
-            tmp = os.path.join(os.path.dirname(path), "." + uuid())
-
-            # The operations order is important here - if some application
-            # would access the file during the process of copyfile then it
-            # would get only the part of file. So, at first, the file should be
-            # copied with the temporary name, and then original file should be
-            # replaced by new.
-            copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
-            remove(path)
-            os.rename(tmp, path)
-
-        else:
-            logger.debug(
-                "Skipping copying for '%s', since it is not "
-                "a symlink or a hardlink.",
-                path,
-            )
-
-        os.chmod(path, self._file_mode)
-
-    def _unprotect_dir(self, path):
-        for fname in self.fs.walk_files(path):
-            self._unprotect_file(fname)
-
-    def unprotect(self, path_info):
-        if not os.path.exists(path_info):
-            from dvc.exceptions import DvcException
-
-            raise DvcException(
-                f"can't unprotect non-existing data '{path_info}'"
-            )
-
-        if os.path.isdir(path_info):
-            self._unprotect_dir(path_info)
-        else:
-            self._unprotect_file(path_info)
-
-    def protect(self, path_info):
-        try:
-            os.chmod(path_info, self.CACHE_MODE)
-        except OSError:
-            # NOTE: not being able to protect cache file is not fatal, it
-            # might happen on funky filesystems (e.g. Samba, see #5255),
-            # read-only filesystems or in a shared cache scenario.
-            logger.trace("failed to protect '%s'", path_info, exc_info=True)
-
-    def is_protected(self, path_info):
-        try:
-            mode = os.stat(path_info).st_mode
-        except FileNotFoundError:
-            return False
-
-        return stat.S_IMODE(mode) == self.CACHE_MODE
-
-    def set_exec(self, path_info):
-        mode = os.stat(path_info).st_mode | stat.S_IEXEC
-        try:
-            os.chmod(path_info, mode)
-        except OSError:
-            logger.trace(
-                "failed to chmod '%s' '%s'",
-                oct(mode),
-                path_info,
-                exc_info=True,
-            )

dvc/objects/errors.py

@@ -1,9 +0,0 @@
-from dvc.exceptions import DvcException
-
-
-class ObjectError(DvcException):
-    pass
-
-
-class ObjectFormatError(ObjectError):
-    pass

dvc/objects/file.py

@@ -1,53 +0,0 @@
-import logging
-
-from .errors import ObjectFormatError
-from .stage import get_file_hash
-
-logger = logging.getLogger(__name__)
-
-
-class HashFile:
-    def __init__(self, path_info, fs, hash_info):
-        self.path_info = path_info
-        self.fs = fs
-        self.hash_info = hash_info
-
-    @property
-    def size(self):
-        if not (self.path_info and self.fs):
-            return None
-        return self.fs.getsize(self.path_info)
-
-    def __len__(self):
-        return 1
-
-    def __str__(self):
-        return f"object {self.hash_info}"
-
-    def __bool__(self):
-        return bool(self.hash_info)
-
-    def __eq__(self, other):
-        if not isinstance(other, HashFile):
-            return False
-        return (
-            self.path_info == other.path_info
-            and self.fs == other.fs
-            and self.hash_info == other.hash_info
-        )
-
-    def check(self, odb):
-        actual = get_file_hash(
-            self.path_info, self.fs, self.hash_info.name, odb.repo.state
-        )
-
-        logger.trace(
-            "cache '%s' expected '%s' actual '%s'",
-            self.path_info,
-            self.hash_info,
-            actual,
-        )
-
-        assert actual.name == self.hash_info.name
-        if actual.value.split(".")[0] != self.hash_info.value.split(".")[0]:
-            raise ObjectFormatError(f"{self} is corrupted")

dvc/objects/stage.py

@@ -1,187 +0,0 @@
-import errno
-import os
-from concurrent.futures import ThreadPoolExecutor
-
-from dvc.exceptions import DvcIgnoreInCollectedDirError
-from dvc.hash_info import HashInfo
-from dvc.ignore import DvcIgnore
-from dvc.progress import Tqdm
-from dvc.utils import file_md5
-
-
-def _get_file_hash(path_info, fs, name):
-    info = fs.info(path_info)
-    if name in info:
-        assert not info[name].endswith(".dir")
-        return HashInfo(name, info[name], size=info["size"])
-
-    func = getattr(fs, name, None)
-    if func:
-        return func(path_info)
-
-    if name == "md5":
-        return HashInfo(
-            name, file_md5(path_info, fs), size=fs.getsize(path_info)
-        )
-
-    raise NotImplementedError
-
-
-def get_file_hash(path_info, fs, name, state=None):
-    if state:
-        hash_info = state.get(  # pylint: disable=assignment-from-none
-            path_info, fs,
-        )
-        if hash_info:
-            return hash_info
-
-    if not fs.exists(path_info):
-        raise FileNotFoundError(
-            errno.ENOENT, os.strerror(errno.ENOENT), path_info
-        )
-
-    hash_info = _get_file_hash(path_info, fs, name)
-
-    if state:
-        assert ".dir" not in hash_info.value
-        state.save(path_info, fs, hash_info)
-
-    return hash_info
-
-
-def _calculate_hashes(path_info, fs, name, state, **kwargs):
-    def _hash(file_info):
-        return file_info, get_file_hash(file_info, fs, name, state)
-
-    with Tqdm(
-        unit="md5", desc="Computing file/dir hashes (only done once)",
-    ) as pbar:
-        worker = pbar.wrap_fn(_hash)
-        with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
-            pairs = executor.map(worker, fs.walk_files(path_info, **kwargs))
-            return dict(pairs)
-
-
-def _iter_hashes(path_info, fs, name, state, **kwargs):
-    if name in fs.DETAIL_FIELDS:
-        for details in fs.ls(path_info, recursive=True, detail=True):
-            file_info = path_info.replace(path=details["name"])
-            hash_info = HashInfo(
-                name, details[name], size=details.get("size"),
-            )
-            yield file_info, hash_info
-
-        return None
-
-    yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items()
-
-
-def _build_tree(path_info, fs, name, state, **kwargs):
-    from .file import HashFile
-    from .tree import Tree
-
-    tree = Tree(None, None, None)
-    for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs):
-        if DvcIgnore.DVCIGNORE_FILE == fi.name:
-            raise DvcIgnoreInCollectedDirError(fi.parent)
-
-        obj = HashFile(fi, fs, hi)
-
-        # NOTE: this is lossy transformation:
-        #   "hey\there" -> "hey/there"
-        #   "hey/there" -> "hey/there"
-        # The latter is fine filename on Windows, which
-        # will transform to dir/file on back transform.
-        #
-        # Yes, this is a BUG, as long as we permit "/" in
-        # filenames on Windows and "\" on Unix
-        tree.add(fi.relative_to(path_info).parts, obj)
-
-    tree.digest()
-
-    return tree
-
-
-def _get_tree_obj(path_info, fs, name, odb, state, **kwargs):
-    from .tree import Tree
-
-    value = fs.info(path_info).get(name)
-    if value:
-        hash_info = HashInfo(name, value)
-        try:
-            return Tree.load(odb, hash_info)
-        except FileNotFoundError:
-            pass
-
-    tree = _build_tree(path_info, fs, name, state, **kwargs)
-
-    odb.add(tree.path_info, tree.fs, tree.hash_info)
-    if name != "md5":
-        # NOTE: used only for external outputs. Initial reasoning was to be
-        # able to validate .dir files right in the workspace (e.g. check s3
-        # etag), but could be dropped for manual validation with regular md5,
-        # that would be universal for all clouds.
-        raw = odb.get(tree.hash_info)
-        hash_info = get_file_hash(raw.path_info, raw.fs, name, state)
-        tree.hash_info.name = hash_info.name
-        tree.hash_info.value = hash_info.value
-        if not tree.hash_info.value.endswith(".dir"):
-            tree.hash_info.value += ".dir"
-        odb.add(tree.path_info, tree.fs, tree.hash_info)
-
-    return tree
-
-
-def stage(odb, path_info, fs, name, **kwargs):
-    assert path_info and (
-        isinstance(path_info, str) or path_info.scheme == fs.scheme
-    )
-
-    if not fs.exists(path_info):
-        raise FileNotFoundError(
-            errno.ENOENT, os.strerror(errno.ENOENT), path_info
-        )
-
-    state = odb.repo.state
-    # pylint: disable=assignment-from-none
-    hash_info = state.get(path_info, fs)
-
-    # If we have dir hash in state db, but dir cache file is lost,
-    # then we need to recollect the dir via .get_dir_hash() call below,
-    # see https://github.com/iterative/dvc/issues/2219 for context
-    if (
-        hash_info
-        and hash_info.isdir
-        and not odb.fs.exists(odb.hash_to_path_info(hash_info.value))
-    ):
-        hash_info = None
-
-    if hash_info:
-        from . import load
-        from .tree import Tree
-
-        obj = load(odb, hash_info)
-        if isinstance(obj, Tree):
-            obj.hash_info.nfiles = len(obj)
-            for key, entry in obj:
-                entry.fs = fs
-                entry.path_info = path_info.joinpath(*key)
-        else:
-            obj.fs = fs
-            obj.path_info = path_info
-        assert obj.hash_info.name == name
-        obj.hash_info.size = hash_info.size
-        return obj
-
-    if fs.isdir(path_info):
-        obj = _get_tree_obj(path_info, fs, name, odb, state, **kwargs)
-    else:
-        from .file import HashFile
-
-        hash_info = get_file_hash(path_info, fs, name, state)
-        obj = HashFile(path_info, fs, hash_info)
-
-    if obj.hash_info and fs.exists(path_info):
-        state.save(path_info, fs, obj.hash_info)
-
-    return obj

dvc/objects/transfer.py

@@ -1,92 +0,0 @@
-import itertools
-import logging
-from concurrent import futures
-
-from dvc.progress import Tqdm
-
-from . import save
-from .file import HashFile
-from .tree import Tree
-
-logger = logging.getLogger(__name__)
-
-
-def _transfer_file(odb, from_fs, from_info):
-    from dvc.utils import tmp_fname
-    from dvc.utils.stream import HashedStreamReader
-
-    tmp_info = odb.fs.path_info / tmp_fname()
-    with from_fs.open(
-        from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE
-    ) as stream:
-        stream_reader = HashedStreamReader(stream)
-        # Since we don't know the hash beforehand, we'll
-        # upload it to a temporary location and then move
-        # it.
-        odb.fs.upload_fobj(
-            stream_reader,
-            tmp_info,
-            total=from_fs.getsize(from_info),
-            desc=from_info.name,
-        )
-
-    hash_info = stream_reader.hash_info
-    return tmp_info, hash_info
-
-
-def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-    rel_path_infos = {}
-    from_infos = from_fs.walk_files(from_info)
-
-    def create_tasks(executor, amount):
-        for entry_info in itertools.islice(from_infos, amount):
-            pbar.total += 1
-            task = executor.submit(
-                pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info
-            )
-            rel_path_infos[task] = entry_info.relative_to(from_info)
-            yield task
-
-    pbar.total = 0
-    with futures.ThreadPoolExecutor(max_workers=jobs) as executor:
-        tasks = set(create_tasks(executor, jobs * 5))
-
-        while tasks:
-            done, tasks = futures.wait(
-                tasks, return_when=futures.FIRST_COMPLETED
-            )
-            tasks.update(create_tasks(executor, len(done)))
-            for task in done:
-                yield rel_path_infos.pop(task), task.result()
-
-
-def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
-    tree = Tree(None, None, None)
-
-    with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
-        for (
-            entry_info,
-            (entry_tmp_info, entry_hash),
-        ) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-            obj = HashFile(entry_tmp_info, odb.fs, entry_hash)
-            tree.add(entry_info.parts, obj)
-
-    tree.digest()
-    save(odb, tree)
-    return tree.hash_info
-
-
-def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
-    jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
-
-    if from_fs.isdir(from_info):
-        return _transfer_directory(
-            odb,
-            from_fs,
-            from_info,
-            jobs=jobs,
-            no_progress_bar=no_progress_bar,
-        )
-    tmp_info, hash_info = _transfer_file(odb, from_fs, from_info)
-    odb.add(tmp_info, odb.fs, hash_info)
-    return hash_info

dvc/objects/tree.py

@@ -1,211 +0,0 @@
-import json
-import logging
-import posixpath
-
-from funcy import cached_property
-
-from .errors import ObjectFormatError
-from .file import HashFile
-from .stage import get_file_hash
-
-logger = logging.getLogger(__name__)
-
-
-class Tree(HashFile):
-    PARAM_RELPATH = "relpath"
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._dict = {}
-
-    @cached_property
-    def trie(self):
-        from pygtrie import Trie
-
-        return Trie(self._dict)
-
-    @property
-    def size(self):
-        try:
-            return sum(obj.size for _, obj in self)
-        except TypeError:
-            return None
-
-    def add(self, key, obj):
-        self.__dict__.pop("trie", None)
-        self._dict[key] = obj
-
-    def digest(self):
-        from dvc.fs.memory import MemoryFileSystem
-        from dvc.path_info import PathInfo
-        from dvc.utils import tmp_fname
-
-        memfs = MemoryFileSystem(None, {})
-        path_info = PathInfo(tmp_fname(""))
-        with memfs.open(path_info, "wb") as fobj:
-            fobj.write(self.as_bytes())
-        self.fs = memfs
-        self.path_info = path_info
-        self.hash_info = get_file_hash(path_info, memfs, "md5")
-        self.hash_info.value += ".dir"
-        self.hash_info.size = self.size
-        self.hash_info.nfiles = len(self)
-
-    def __len__(self):
-        return len(self._dict)
-
-    def __iter__(self):
-        yield from self._dict.items()
-
-    def as_dict(self):
-        return self._dict.copy()
-
-    def as_list(self):
-        from operator import itemgetter
-
-        # Sorting the list by path to ensure reproducibility
-        return sorted(
-            (
-                {
-                    # NOTE: not using hash_info.to_dict() because we don't want
-                    # size/nfiles fields at this point.
-                    obj.hash_info.name: obj.hash_info.value,
-                    self.PARAM_RELPATH: posixpath.sep.join(parts),
-                }
-                for parts, obj in self._dict.items()  # noqa: B301
-            ),
-            key=itemgetter(self.PARAM_RELPATH),
-        )
-
-    def as_bytes(self):
-        return json.dumps(self.as_list(), sort_keys=True).encode("utf-8")
-
-    @classmethod
-    def from_list(cls, lst):
-        from dvc.hash_info import HashInfo
-
-        tree = cls(None, None, None)
-        for _entry in lst:
-            entry = _entry.copy()
-            relpath = entry.pop(cls.PARAM_RELPATH)
-            parts = tuple(relpath.split(posixpath.sep))
-            hash_info = HashInfo.from_dict(entry)
-            obj = HashFile(None, None, hash_info)
-            tree.add(parts, obj)
-        return tree
-
-    @classmethod
-    def load(cls, odb, hash_info):
-
-        obj = odb.get(hash_info)
-
-        try:
-            with obj.fs.open(obj.path_info, "r") as fobj:
-                raw = json.load(fobj)
-        except ValueError as exc:
-            raise ObjectFormatError(f"{obj} is corrupted") from exc
-
-        if not isinstance(raw, list):
-            logger.error(
-                "dir cache file format error '%s' [skipping the file]",
-                obj.path_info,
-            )
-            raise ObjectFormatError(f"{obj} is corrupted")
-
-        tree = cls.from_list(raw)
-        tree.path_info = obj.path_info
-        tree.fs = obj.fs
-        tree.hash_info = hash_info
-
-        return tree
-
-    def filter(self, odb, prefix):
-        obj = self._dict.get(prefix)
-        if obj:
-            return obj
-
-        depth = len(prefix)
-        tree = Tree(None, None, None)
-        try:
-            for key, obj in self.trie.items(prefix):
-                tree.add(key[depth:], obj)
-        except KeyError:
-            return None
-        tree.digest()
-        odb.add(tree.path_info, tree.fs, tree.hash_info)
-        return tree
-
-
-def _get_dir_size(odb, tree):
-    try:
-        return sum(
-            odb.fs.getsize(odb.hash_to_path_info(obj.hash_info.value))
-            for _, obj in tree
-        )
-    except FileNotFoundError:
-        return None
-
-
-def _diff(ancestor, other, allow_removed=False):
-    from dictdiffer import diff
-
-    from dvc.exceptions import MergeError
-
-    allowed = ["add"]
-    if allow_removed:
-        allowed.append("remove")
-
-    result = list(diff(ancestor, other))
-    for typ, _, _ in result:
-        if typ not in allowed:
-            raise MergeError(
-                "unable to auto-merge directories with diff that contains "
-                f"'{typ}'ed files"
-            )
-    return result
-
-
-def _merge(ancestor, our, their):
-    import copy
-
-    from dictdiffer import patch
-
-    our_diff = _diff(ancestor, our)
-    if not our_diff:
-        return copy.deepcopy(their)
-
-    their_diff = _diff(ancestor, their)
-    if not their_diff:
-        return copy.deepcopy(our)
-
-    # make sure there are no conflicting files
-    _diff(our, their, allow_removed=True)
-
-    return patch(our_diff + their_diff, ancestor)
-
-
-def merge(odb, ancestor_info, our_info, their_info):
-    from . import load
-
-    assert our_info
-    assert their_info
-
-    if ancestor_info:
-        ancestor = load(odb, ancestor_info)
-    else:
-        ancestor = Tree(None, None, None)
-
-    our = load(odb, our_info)
-    their = load(odb, their_info)
-
-    merged_dict = _merge(ancestor.as_dict(), our.as_dict(), their.as_dict(),)
-
-    merged = Tree(None, None, None)
-    for key, hi in merged_dict.items():
-        merged.add(key, hi)
-    merged.digest()
-
-    odb.add(merged.path_info, merged.fs, merged.hash_info)
-    hash_info = merged.hash_info
-    hash_info.size = _get_dir_size(odb, merged)
-    return hash_info

dvc/output/__init__.py

@@ -6,6 +6,7 @@ from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
 
 from dvc.hash_info import HashInfo
 from dvc.output.base import BaseOutput
+from dvc.output.gs import GSOutput
 from dvc.output.hdfs import HDFSOutput
 from dvc.output.local import LocalOutput
 from dvc.output.s3 import S3Output
@@ -13,15 +14,25 @@ from dvc.output.ssh import SSHOutput
 from dvc.output.webhdfs import WebHDFSOutput
 from dvc.scheme import Schemes
 
-from ..fs import get_cloud_fs
-from ..fs.hdfs import HDFSFileSystem
-from ..fs.local import LocalFileSystem
-from ..fs.s3 import S3FileSystem
-from ..fs.webhdfs import WebHDFSFileSystem
+from ..tree import get_cloud_tree
+from ..tree.hdfs import HDFSTree
+from ..tree.local import LocalTree
+from ..tree.s3 import S3Tree
+from ..tree.webhdfs import WebHDFSTree
+
+OUTS = [
+    HDFSOutput,
+    S3Output,
+    GSOutput,
+    SSHOutput,
+    WebHDFSOutput,
+    # NOTE: LocalOutput is the default choice
+]
 
 OUTS_MAP = {
     Schemes.HDFS: HDFSOutput,
     Schemes.S3: S3Output,
+    Schemes.GS: GSOutput,
     Schemes.SSH: SSHOutput,
     Schemes.LOCAL: LocalOutput,
     Schemes.WEBHDFS: WebHDFSOutput,
@@ -35,17 +46,17 @@ CHECKSUM_SCHEMA = Any(
 
 # NOTE: currently there are only 3 possible checksum names:
 #
-#    1) md5 (LOCAL, SSH);
+#    1) md5 (LOCAL, SSH, GS);
 #    2) etag (S3);
 #    3) checksum (HDFS);
 #
 # so when a few types of outputs share the same name, we only need
 # specify it once.
 CHECKSUMS_SCHEMA = {
-    LocalFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    S3FileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    HDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
-    WebHDFSFileSystem.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    WebHDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }
 
 SCHEMA = CHECKSUMS_SCHEMA.copy()
@@ -58,7 +69,6 @@ SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool
 SCHEMA[HashInfo.PARAM_SIZE] = int
 SCHEMA[HashInfo.PARAM_NFILES] = int
 SCHEMA[BaseOutput.PARAM_DESC] = str
-SCHEMA[BaseOutput.PARAM_ISEXEC] = bool
 
 
 def _get(
@@ -70,43 +80,50 @@ def _get(
     plot=False,
     persist=False,
     checkpoint=False,
-    live=False,
     desc=None,
-    isexec=False,
 ):
     parsed = urlparse(p)
 
     if parsed.scheme == "remote":
-        fs = get_cloud_fs(stage.repo, name=parsed.netloc)
-        return OUTS_MAP[fs.scheme](
+        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
+        return OUTS_MAP[tree.scheme](
             stage,
             p,
             info,
             cache=cache,
-            fs=fs,
+            tree=tree,
             metric=metric,
             plot=plot,
             persist=persist,
             checkpoint=checkpoint,
-            live=live,
             desc=desc,
-            isexec=isexec,
         )
 
-    out_cls = OUTS_MAP.get(parsed.scheme, LocalOutput)
-    return out_cls(
+    for o in OUTS:
+        if o.supported(p):
+            return o(
+                stage,
+                p,
+                info,
+                cache=cache,
+                tree=None,
+                metric=metric,
+                plot=plot,
+                persist=persist,
+                checkpoint=checkpoint,
+                desc=desc,
+            )
+    return LocalOutput(
         stage,
         p,
         info,
         cache=cache,
-        fs=None,
+        tree=None,
         metric=metric,
         plot=plot,
         persist=persist,
         checkpoint=checkpoint,
-        live=live,
         desc=desc,
-        isexec=isexec,
     )
 
 
@@ -120,8 +137,6 @@ def loadd_from(stage, d_list):
         persist = d.pop(BaseOutput.PARAM_PERSIST, False)
         checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False)
         desc = d.pop(BaseOutput.PARAM_DESC, False)
-        isexec = d.pop(BaseOutput.PARAM_ISEXEC, False)
-        live = d.pop(BaseOutput.PARAM_LIVE, False)
         ret.append(
             _get(
                 stage,
@@ -133,8 +148,6 @@ def loadd_from(stage, d_list):
                 persist=persist,
                 checkpoint=checkpoint,
                 desc=desc,
-                isexec=isexec,
-                live=live,
             )
         )
     return ret
@@ -148,8 +161,6 @@ def loads_from(
     plot=False,
     persist=False,
     checkpoint=False,
-    isexec=False,
-    live=False,
 ):
     return [
         _get(
@@ -161,8 +172,6 @@ def loads_from(
             plot=plot,
             persist=persist,
             checkpoint=checkpoint,
-            isexec=isexec,
-            live=live,
         )
         for s in s_list
     ]
@@ -191,35 +200,21 @@ def _merge_data(s_list):
 
 
 @collecting
-def load_from_pipeline(stage, data, typ="outs"):
-    if typ not in (
-        stage.PARAM_OUTS,
-        stage.PARAM_METRICS,
-        stage.PARAM_PLOTS,
-        stage.PARAM_LIVE,
-    ):
+def load_from_pipeline(stage, s_list, typ="outs"):
+    if typ not in (stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS):
         raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
 
     metric = typ == stage.PARAM_METRICS
     plot = typ == stage.PARAM_PLOTS
-    live = typ == stage.PARAM_LIVE
 
-    if live:
-        # `live` is single object
-        data = [data]
-
-    d = _merge_data(data)
+    d = _merge_data(s_list)
 
     for path, flags in d.items():
-        plt_d, live_d = {}, {}
+        plt_d = {}
         if plot:
             from dvc.schema import PLOT_PROPS
 
             plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
-        if live:
-            from dvc.schema import LIVE_PROPS
-
-            live_d, flags = _split_dict(flags, keys=LIVE_PROPS.keys())
         extra = project(
             flags,
             [
@@ -228,13 +223,4 @@ def load_from_pipeline(stage, data, typ="outs"):
                 BaseOutput.PARAM_CHECKPOINT,
             ],
         )
-
-        yield _get(
-            stage,
-            path,
-            {},
-            plot=plt_d or plot,
-            metric=metric,
-            live=live_d or live,
-            **extra,
-        )
+        yield _get(stage, path, {}, plot=plt_d or plot, metric=metric, **extra)

dvc/output/base.py

@@ -6,9 +6,8 @@ from urllib.parse import urlparse
 
 from voluptuous import Any
 
-import dvc.objects as objects
 import dvc.prompt as prompt
-from dvc.checkout import checkout
+from dvc.cache import NamedCache
 from dvc.exceptions import (
     CheckoutError,
     CollectCacheError,
@@ -17,11 +16,8 @@ from dvc.exceptions import (
     RemoteCacheRequiredError,
 )
 from dvc.hash_info import HashInfo
-from dvc.objects.db import NamedCache
-from dvc.objects.errors import ObjectFormatError
-from dvc.objects.stage import stage as ostage
 
-from ..fs.base import BaseFileSystem
+from ..tree.base import BaseTree
 
 logger = logging.getLogger(__name__)
 
@@ -50,7 +46,7 @@ class OutputAlreadyTrackedError(DvcException):
 
 class OutputIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__(f"DVC file '{path}' cannot be an output.")
+        super().__init__(f"Stage file '{path}' cannot be an output.")
 
 
 class OutputIsIgnoredError(DvcException):
@@ -62,7 +58,7 @@ class OutputIsIgnoredError(DvcException):
 class BaseOutput:
     IS_DEPENDENCY = False
 
-    FS_CLS = BaseFileSystem
+    TREE_CLS = BaseTree
 
     PARAM_PATH = "path"
     PARAM_CACHE = "cache"
@@ -80,10 +76,6 @@ class BaseOutput:
     PARAM_PLOT_HEADER = "header"
     PARAM_PERSIST = "persist"
     PARAM_DESC = "desc"
-    PARAM_ISEXEC = "isexec"
-    PARAM_LIVE = "live"
-    PARAM_LIVE_SUMMARY = "summary"
-    PARAM_LIVE_HTML = "html"
 
     METRIC_SCHEMA = Any(
         None,
@@ -106,21 +98,19 @@ class BaseOutput:
         stage,
         path,
         info=None,
-        fs=None,
+        tree=None,
         cache=True,
         metric=False,
         plot=False,
         persist=False,
         checkpoint=False,
-        live=False,
         desc=None,
-        isexec=False,
     ):
         self._validate_output_path(path, stage)
         # This output (and dependency) objects have too many paths/urls
         # here is a list and comments:
         #
-        #   .def_path - path from definition in DVC file
+        #   .def_path - path from definition in stage file
         #   .path_info - PathInfo/URLInfo structured resolved path
         #   .fspath - local only, resolved
         #   .__str__ - for presentation purposes, def_path/relpath
@@ -131,30 +121,26 @@ class BaseOutput:
         self.repo = stage.repo if stage else None
         self.def_path = path
         self.hash_info = HashInfo.from_dict(info)
-        if fs:
-            self.fs = fs
+        if tree:
+            self.tree = tree
         else:
-            self.fs = self.FS_CLS(self.repo, {})
+            self.tree = self.TREE_CLS(self.repo, {})
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
         self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
         self.checkpoint = checkpoint
-        self.live = live
         self.desc = desc
 
-        self.path_info = self._parse_path(fs, path)
-        if self.use_cache and self.odb is None:
+        self.path_info = self._parse_path(tree, path)
+        if self.use_cache and self.cache is None:
             raise RemoteCacheRequiredError(self.path_info)
 
-        self.obj = None
-        self.isexec = False if self.IS_DEPENDENCY else isexec
-
-    def _parse_path(self, fs, path):
-        if fs:
+    def _parse_path(self, tree, path):
+        if tree:
             parsed = urlparse(path)
-            return fs.path_info / parsed.path.lstrip("/")
-        return self.FS_CLS.PATH_CLS(path)
+            return tree.path_info / parsed.path.lstrip("/")
+        return self.TREE_CLS.PATH_CLS(path)
 
     def __repr__(self):
         return "{class_name}: '{def_path}'".format(
@@ -166,7 +152,7 @@ class BaseOutput:
 
     @property
     def scheme(self):
-        return self.FS_CLS.scheme
+        return self.TREE_CLS.scheme
 
     @property
     def is_in_repo(self):
@@ -180,24 +166,25 @@ class BaseOutput:
         return self.use_cache or self.stage.is_repo_import
 
     @property
-    def odb(self):
-        return getattr(self.repo.odb, self.scheme)
+    def cache(self):
+        return getattr(self.repo.cache, self.scheme)
+
+    @property
+    def dir_cache(self):
+        return self.cache.get_dir_cache(self.hash_info)
+
+    @classmethod
+    def supported(cls, url):
+        return cls.TREE_CLS.supported(url)
 
     @property
     def cache_path(self):
-        return self.odb.hash_to_path_info(self.hash_info.value).url
+        return self.cache.tree.hash_to_path_info(self.hash_info.value).url
 
     def get_hash(self):
         if not self.use_cache:
-            return ostage(
-                self.repo.odb.local,
-                self.path_info,
-                self.fs,
-                self.fs.PARAM_CHECKSUM,
-            ).hash_info
-        return ostage(
-            self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
-        ).hash_info
+            return self.tree.get_hash(self.path_info)
+        return self.cache.get_hash(self.tree, self.path_info)
 
     @property
     def is_dir_checksum(self):
@@ -205,7 +192,7 @@ class BaseOutput:
 
     @property
     def exists(self):
-        return self.fs.exists(self.path_info)
+        return self.tree.exists(self.path_info)
 
     def changed_checksum(self):
         return self.hash_info != self.get_hash()
@@ -214,15 +201,9 @@ class BaseOutput:
         if not self.use_cache or not self.hash_info:
             return True
 
-        obj = self.get_obj(filter_info=filter_info)
-        if not obj:
-            return True
-
-        try:
-            objects.check(self.odb, obj)
-            return False
-        except (FileNotFoundError, ObjectFormatError):
-            return True
+        return self.cache.changed_cache(
+            self.hash_info, filter_info=filter_info
+        )
 
     def workspace_status(self):
         if not self.exists:
@@ -249,13 +230,13 @@ class BaseOutput:
 
     @property
     def is_empty(self):
-        return self.fs.is_empty(self.path_info)
+        return self.tree.is_empty(self.path_info)
 
     def isdir(self):
-        return self.fs.isdir(self.path_info)
+        return self.tree.isdir(self.path_info)
 
     def isfile(self):
-        return self.fs.isfile(self.path_info)
+        return self.tree.isfile(self.path_info)
 
     # pylint: disable=no-member
 
@@ -305,38 +286,15 @@ class BaseOutput:
             logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
-        self.obj = ostage(
-            self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
-        )
-        self.hash_info = self.obj.hash_info
-        self.isexec = self.isfile() and self.fs.isexec(self.path_info)
+        self.hash_info = self.get_hash()
 
-    def set_exec(self):
-        if self.isfile() and self.isexec:
-            self.odb.set_exec(self.path_info)
-
-    def commit(self, filter_info=None):
+    def commit(self):
         if not self.exists:
             raise self.DoesNotExistError(self)
 
         assert self.hash_info
-
         if self.use_cache:
-            obj = ostage(
-                self.odb,
-                filter_info or self.path_info,
-                self.fs,
-                self.odb.fs.PARAM_CHECKSUM,
-            )
-            objects.save(self.odb, obj)
-            checkout(
-                filter_info or self.path_info,
-                self.fs,
-                obj,
-                self.odb,
-                relink=True,
-            )
-            self.set_exec()
+            self.cache.save(self.path_info, self.cache.tree, self.hash_info)
 
     def dumpd(self):
         ret = copy(self.hash_info.to_dict())
@@ -370,36 +328,13 @@ class BaseOutput:
         if self.checkpoint:
             ret[self.PARAM_CHECKPOINT] = self.checkpoint
 
-        if self.isexec:
-            ret[self.PARAM_ISEXEC] = self.isexec
-
-        if self.live:
-            ret[self.PARAM_LIVE] = self.live
-
         return ret
 
     def verify_metric(self):
         raise DvcException(f"verify metric is not supported for {self.scheme}")
 
-    def download(self, to, jobs=None):
-        self.fs.download(self.path_info, to.path_info, jobs=jobs)
-
-    def get_obj(self, filter_info=None):
-        if self.obj:
-            obj = self.obj
-        elif self.hash_info:
-            try:
-                obj = objects.load(self.odb, self.hash_info)
-            except FileNotFoundError:
-                return None
-        else:
-            return None
-
-        if filter_info and filter_info != self.path_info:
-            prefix = filter_info.relative_to(self.path_info).parts
-            obj = obj.filter(self.odb, prefix)
-
-        return obj
+    def download(self, to):
+        self.tree.download(self.path_info, to.path_info)
 
     def checkout(
         self,
@@ -408,7 +343,6 @@ class BaseOutput:
         relink=False,
         filter_info=None,
         allow_missing=False,
-        checkpoint_reset=False,
         **kwargs,
     ):
         if not self.use_cache:
@@ -418,38 +352,23 @@ class BaseOutput:
                 )
             return None
 
-        obj = self.get_obj(filter_info=filter_info)
-        if not obj and (filter_info and filter_info != self.path_info):
-            # backward compatibility
-            return None
-
-        if self.checkpoint and checkpoint_reset:
-            if self.exists:
-                self.remove()
-            return None
-
-        added = not self.exists
-
         try:
-            modified = checkout(
-                filter_info or self.path_info,
-                self.fs,
-                obj,
-                self.odb,
+            return self.cache.checkout(
+                self.path_info,
+                self.hash_info,
                 force=force,
                 progress_callback=progress_callback,
                 relink=relink,
+                filter_info=filter_info,
                 **kwargs,
             )
         except CheckoutError:
             if allow_missing or self.checkpoint:
                 return None
             raise
-        self.set_exec()
-        return added, False if added else modified
 
     def remove(self, ignore_remove=False):
-        self.fs.remove(self.path_info)
+        self.tree.remove(self.path_info)
         if self.scheme != "local":
             return
 
@@ -461,7 +380,7 @@ class BaseOutput:
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore_remove(self.fspath)
 
-        self.fs.move(self.path_info, out.path_info)
+        self.tree.move(self.path_info, out.path_info)
         self.def_path = out.def_path
         self.path_info = out.path_info
         self.save()
@@ -471,42 +390,30 @@ class BaseOutput:
             self.repo.scm.ignore(self.fspath)
 
     def get_files_number(self, filter_info=None):
-        if not self.use_cache or not self.hash_info:
+        if not self.use_cache:
             return 0
 
-        if not self.hash_info.isdir:
-            return 1
-
-        if not filter_info or filter_info == self.path_info:
-            return self.hash_info.nfiles or 0
-
-        obj = self.get_obj(filter_info=filter_info)
-        return len(obj) if obj else 0
+        return self.cache.get_files_number(
+            self.path_info, self.hash_info, filter_info
+        )
 
     def unprotect(self):
         if self.exists:
-            self.odb.unprotect(self.path_info)
+            self.tree.unprotect(self.path_info)
 
     def get_dir_cache(self, **kwargs):
-
         if not self.is_dir_checksum:
             raise DvcException("cannot get dir cache for file checksum")
-
-        try:
-            objects.check(self.odb, self.odb.get(self.hash_info))
-        except (FileNotFoundError, ObjectFormatError):
+        if self.cache.changed_cache_file(self.hash_info):
             self.repo.cloud.pull(
                 NamedCache.make("local", self.hash_info.value, str(self)),
                 show_checksums=False,
                 **kwargs,
             )
-
-        try:
-            self.obj = objects.load(self.odb, self.hash_info)
-        except (FileNotFoundError, ObjectFormatError):
-            self.obj = None
-
-        return self.obj
+            self.hash_info.dir_info = None
+        if not self.hash_info.dir_info:
+            self.cache.set_dir_info(self.hash_info)
+        return self.dir_cache
 
     def collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
@@ -537,9 +444,7 @@ class BaseOutput:
         except DvcException:
             logger.debug(f"failed to pull cache for '{self}'")
 
-        try:
-            objects.check(self.odb, self.odb.get(self.hash_info))
-        except (FileNotFoundError, ObjectFormatError):
+        if self.cache.changed_cache_file(self.hash_info):
             msg = (
                 "Missing cache for directory '{}'. "
                 "Cache for files inside will be lost. "
@@ -550,18 +455,22 @@ class BaseOutput:
                     "unable to fully collect used cache"
                     " without cache for directory '{}'".format(self)
                 )
-            return cache
+            else:
+                return cache
 
         path = str(self.path_info)
         filter_path = str(filter_info) if filter_info else None
-        for entry_key, entry_obj in self.obj:
-            entry_path = os.path.join(path, *entry_key)
+        is_win = os.name == "nt"
+        for entry_relpath, entry_hash_info in self.dir_cache.items():
+            if is_win:
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            entry_path = os.path.join(path, entry_relpath)
             if (
                 not filter_path
                 or entry_path == filter_path
                 or entry_path.startswith(filter_path + os.sep)
             ):
-                cache.add(self.scheme, entry_obj.hash_info.value, entry_path)
+                cache.add(self.scheme, entry_hash_info.value, entry_path)
 
         return cache
 
@@ -621,9 +530,8 @@ class BaseOutput:
             raise cls.IsStageFileError(path)
 
         if stage:
-            abs_path = os.path.join(stage.wdir, path)
-            if stage.repo.fs.dvcignore.is_ignored(abs_path):
-                check = stage.repo.fs.dvcignore.check_ignore(abs_path)
+            check = stage.repo.tree.dvcignore.check_ignore(path)
+            if check.match:
                 raise cls.IsIgnoredError(check)
 
     def _check_can_merge(self, out):
@@ -634,7 +542,7 @@ class BaseOutput:
         other = out.dumpd()
 
         ignored = [
-            self.fs.PARAM_CHECKSUM,
+            self.tree.PARAM_CHECKSUM,
             HashInfo.PARAM_SIZE,
             HashInfo.PARAM_NFILES,
         ]
@@ -654,8 +562,6 @@ class BaseOutput:
             )
 
     def merge(self, ancestor, other):
-        from dvc.objects.tree import merge
-
         assert other
 
         if ancestor:
@@ -667,6 +573,6 @@ class BaseOutput:
         self._check_can_merge(self)
         self._check_can_merge(other)
 
-        self.hash_info = merge(
-            self.odb, ancestor_info, self.hash_info, other.hash_info
+        self.hash_info = self.cache.merge(
+            ancestor_info, self.hash_info, other.hash_info
         )

dvc/output/gs.py

@@ -0,0 +1,7 @@
+from dvc.output.base import BaseOutput
+
+from ..tree.gs import GSTree
+
+
+class GSOutput(BaseOutput):
+    TREE_CLS = GSTree

dvc/output/hdfs.py

@@ -1,7 +1,7 @@
 from dvc.output.base import BaseOutput
 
-from ..fs.hdfs import HDFSFileSystem
+from ..tree.hdfs import HDFSTree
 
 
 class HDFSOutput(BaseOutput):
-    FS_CLS = HDFSFileSystem
+    TREE_CLS = HDFSTree

dvc/output/local.py

@@ -8,13 +8,13 @@ from dvc.output.base import BaseOutput
 from dvc.utils import relpath
 from dvc.utils.fs import path_isin
 
-from ..fs.local import LocalFileSystem
+from ..tree.local import LocalTree
 
 logger = logging.getLogger(__name__)
 
 
 class LocalOutput(BaseOutput):
-    FS_CLS = LocalFileSystem
+    TREE_CLS = LocalTree
     sep = os.sep
 
     def __init__(self, stage, path, *args, **kwargs):
@@ -25,14 +25,14 @@ class LocalOutput(BaseOutput):
         if (
             self.is_in_repo
             and self.repo
-            and isinstance(self.repo.fs, LocalFileSystem)
+            and isinstance(self.repo.tree, LocalTree)
         ):
-            self.fs = self.repo.fs
+            self.tree = self.repo.tree
 
-    def _parse_path(self, fs, path):
+    def _parse_path(self, tree, path):
         parsed = urlparse(path)
         if parsed.scheme == "remote":
-            p = fs.path_info / parsed.path.lstrip("/")
+            p = tree.path_info / parsed.path.lstrip("/")
         else:
             # NOTE: we can path either from command line or .dvc file,
             # so we should expect both posix and windows style paths.
@@ -40,12 +40,12 @@ class LocalOutput(BaseOutput):
             #
             # FIXME: if we have Windows path containing / or posix one with \
             # then we have #2059 bug and can't really handle that.
-            p = self.FS_CLS.PATH_CLS(path)
+            p = self.TREE_CLS.PATH_CLS(path)
             if self.stage and not p.is_absolute():
                 p = self.stage.wdir / p
 
         abs_p = os.path.abspath(os.path.normpath(p))
-        return self.FS_CLS.PATH_CLS(abs_p)
+        return self.TREE_CLS.PATH_CLS(abs_p)
 
     def __str__(self):
         if not self.repo or not self.is_in_repo:
@@ -91,6 +91,6 @@ class LocalOutput(BaseOutput):
             logger.debug(msg, str(self.path_info), name)
             return
 
-        if not istextfile(path, self.fs):
+        if not istextfile(path):
             msg = "binary file '{}' cannot be used as {}."
             raise DvcException(msg.format(self.path_info, name))

dvc/output/s3.py

@@ -1,7 +1,7 @@
 from dvc.output.base import BaseOutput
 
-from ..fs.s3 import S3FileSystem
+from ..tree.s3 import S3Tree
 
 
 class S3Output(BaseOutput):
-    FS_CLS = S3FileSystem
+    TREE_CLS = S3Tree

dvc/output/ssh.py

@@ -1,7 +1,7 @@
 from dvc.output.base import BaseOutput
 
-from ..fs.ssh import SSHFileSystem
+from ..tree.ssh import SSHTree
 
 
 class SSHOutput(BaseOutput):
-    FS_CLS = SSHFileSystem
+    TREE_CLS = SSHTree

dvc/output/webhdfs.py

@@ -1,7 +1,7 @@
 from dvc.output.base import BaseOutput
 
-from ..fs.webhdfs import WebHDFSFileSystem
+from ..tree.webhdfs import WebHDFSTree
 
 
 class WebHDFSOutput(BaseOutput):
-    FS_CLS = WebHDFSFileSystem
+    TREE_CLS = WebHDFSTree

dvc/parsing/__init__.py

@@ -1,20 +1,14 @@
 import logging
+import os
+from collections import defaultdict
 from collections.abc import Mapping, Sequence
 from copy import deepcopy
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    List,
-    NamedTuple,
-    Optional,
-    Tuple,
-    Type,
-    Union,
-)
+from itertools import starmap
+from typing import TYPE_CHECKING, Dict, List, Optional
 
-from funcy import cached_property, collecting, first, isa, join, reraise
+from funcy import join, lfilter
 
+from dvc.dependency.param import ParamsDependency
 from dvc.exceptions import DvcException
 from dvc.parsing.interpolate import ParseError
 from dvc.path_info import PathInfo
@@ -25,20 +19,12 @@ from .context import (
     ContextError,
     KeyNotInContext,
     MergeError,
+    Meta,
     Node,
-    SeqOrMap,
-    VarsAlreadyLoaded,
-)
-from .interpolate import (
-    check_recursive_parse_errors,
-    is_interpolated_string,
-    recurse,
-    to_str,
+    SetError,
 )
 
 if TYPE_CHECKING:
-    from typing_extensions import NoReturn
-
     from dvc.repo import Repo
 
 logger = logging.getLogger(__name__)
@@ -46,29 +32,30 @@ logger = logging.getLogger(__name__)
 STAGES_KWD = "stages"
 VARS_KWD = "vars"
 WDIR_KWD = "wdir"
+DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
 PARAMS_KWD = "params"
 FOREACH_KWD = "foreach"
 DO_KWD = "do"
+SET_KWD = "set"
 
-DEFAULT_PARAMS_FILE = "params.yaml"
+DEFAULT_SENTINEL = object()
 
 JOIN = "@"
-DictStr = Dict[str, Any]
+
+
+class VarsAlreadyLoaded(DvcException):
+    pass
 
 
 class ResolveError(DvcException):
     pass
 
 
-class EntryNotFound(DvcException):
-    pass
-
-
-def _format_preamble(msg: str, path: str, spacing: str = " ") -> str:
+def _format_preamble(msg, path, spacing=" "):
     return f"failed to parse {msg} in '{path}':{spacing}"
 
 
-def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn":
+def format_and_raise(exc, msg, path):
     spacing = (
         "\n"
         if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded))
@@ -81,154 +68,165 @@ def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn":
     _reraise_err(ResolveError, message, from_exc=exc)
 
 
-def _reraise_err(
-    exc_cls: Type[Exception], *args, from_exc: Exception = None
-) -> "NoReturn":
+def _reraise_err(exc_cls, *args, from_exc=None):
     err = exc_cls(*args)
     if from_exc and logger.isEnabledFor(logging.DEBUG):
         raise err from from_exc
     raise err
 
 
-def check_syntax_errors(
-    definition: DictStr, name: str, path: str, where: str = "stages"
-):
-    for key, d in definition.items():
-        try:
-            check_recursive_parse_errors(d)
-        except ParseError as exc:
-            format_and_raise(exc, f"'{where}.{name}.{key}'", path)
-
-
-def is_map_or_seq(data: Any) -> bool:
-    _is_map_or_seq = isa(Mapping, Sequence)
-    return not isinstance(data, str) and _is_map_or_seq(data)
-
-
-def split_foreach_name(name: str) -> Tuple[str, Optional[str]]:
-    group, *keys = name.rsplit(JOIN, maxsplit=1)
-    return group, first(keys)
-
-
-def check_interpolations(data: DictStr, where: str, path: str):
-    def func(s: DictStr) -> None:
-        if is_interpolated_string(s):
-            raise ResolveError(
-                _format_preamble(f"'{where}'", path)
-                + "interpolating is not allowed"
-            )
-
-    return recurse(func)(data)
-
-
-Definition = Union["ForeachDefinition", "EntryDefinition"]
-
-
-def make_definition(
-    resolver: "DataResolver", name: str, definition: DictStr, **kwargs
-) -> Definition:
-    args = resolver, resolver.context, name, definition
-    if FOREACH_KWD in definition:
-        return ForeachDefinition(*args, **kwargs)
-    return EntryDefinition(*args, **kwargs)
-
-
 class DataResolver:
     def __init__(self, repo: "Repo", wdir: PathInfo, d: dict):
-        self.fs = fs = repo.fs
+        self.data: dict = d
         self.wdir = wdir
+        self.repo = repo
+        self.tree = self.repo.tree
+        self.imported_files: Dict[str, Optional[List[str]]] = {}
         self.relpath = relpath(self.wdir / "dvc.yaml")
 
-        vars_ = d.get(VARS_KWD, [])
-        check_interpolations(vars_, VARS_KWD, self.relpath)
-        self.context: Context = Context()
+        to_import: PathInfo = wdir / DEFAULT_PARAMS_FILE
+        if self.tree.exists(to_import):
+            self.imported_files = {os.path.abspath(to_import): None}
+            self.global_ctx = Context.load_from(self.tree, to_import)
+        else:
+            self.global_ctx = Context()
+            logger.debug(
+                "%s does not exist, it won't be used in parametrization",
+                to_import,
+            )
 
+        vars_ = d.get(VARS_KWD, [])
         try:
-            args = fs, vars_, wdir  # load from `vars` section
-            self.context.load_from_vars(*args, default=DEFAULT_PARAMS_FILE)
-        except ContextError as exc:
+            self.load_from_vars(
+                self.global_ctx, vars_, wdir, skip_imports=self.imported_files
+            )
+        except (ContextError, VarsAlreadyLoaded) as exc:
             format_and_raise(exc, "'vars'", self.relpath)
 
-        # we use `tracked_vars` to keep a dictionary of used variables
-        # by the interpolated entries.
-        self.tracked_vars: Dict[str, Mapping] = {}
+        self.tracked_vars = {}
 
-        stages_data = d.get(STAGES_KWD, {})
-        # we wrap the definitions into ForeachDefinition and EntryDefinition,
-        # that helps us to optimize, cache and selectively load each one of
-        # them as we need, and simplify all of this DSL/parsing logic.
-        self.definitions: Dict[str, Definition] = {
-            name: make_definition(self, name, definition)
-            for name, definition in stages_data.items()
-        }
+    @staticmethod
+    def check_loaded(path, item, keys, skip_imports):
+        if not keys and isinstance(skip_imports[path], list):
+            raise VarsAlreadyLoaded(
+                f"cannot load '{item}' as it's partially loaded already"
+            )
+        elif keys and skip_imports[path] is None:
+            raise VarsAlreadyLoaded(
+                f"cannot partially load '{item}' as it's already loaded."
+            )
+        elif keys and isinstance(skip_imports[path], list):
+            if not set(keys).isdisjoint(set(skip_imports[path])):
+                raise VarsAlreadyLoaded(
+                    f"cannot load '{item}' as it's partially loaded already"
+                )
 
-    def resolve_one(self, name: str):
-        group, key = split_foreach_name(name)
+    def load_from_vars(
+        self,
+        context: "Context",
+        vars_: List,
+        wdir: PathInfo,
+        skip_imports: Dict[str, Optional[List[str]]],
+        stage_name: str = None,
+    ):
+        stage_name = stage_name or ""
+        for index, item in enumerate(vars_):
+            assert isinstance(item, (str, dict))
+            if isinstance(item, str):
+                path, _, keys_str = item.partition(":")
+                keys = lfilter(bool, keys_str.split(","))
 
-        if not self._has_group_and_key(group, key):
-            raise EntryNotFound(f"Could not find '{name}'")
+                path_info = wdir / path
+                path = os.path.abspath(path_info)
 
-        # all of the checks for `key` not being None for `ForeachDefinition`
-        # and/or `group` not existing in the `interim`, etc. should be
-        # handled by the `self.has_key()` above.
-        definition = self.definitions[group]
-        if isinstance(definition, EntryDefinition):
-            return definition.resolve()
+                if path in skip_imports:
+                    if not keys and skip_imports[path] is None:
+                        # allow specifying complete filepath multiple times
+                        continue
+                    self.check_loaded(path, item, keys, skip_imports)
 
-        assert key
-        return definition.resolve_one(key)
+                context.merge_from(self.tree, path_info, select_keys=keys)
+                skip_imports[path] = keys if keys else None
+            else:
+                joiner = "." if stage_name else ""
+                meta = Meta(source=f"{stage_name}{joiner}vars[{index}]")
+                context.merge_update(Context(item, meta=meta))
+
+    def _resolve_entry(self, name: str, definition):
+        context = Context.clone(self.global_ctx)
+        if FOREACH_KWD in definition:
+            assert DO_KWD in definition
+            self.set_context_from(
+                context, definition.get(SET_KWD, {}), source=[name, "set"]
+            )
+            return self._foreach(
+                context, name, definition[FOREACH_KWD], definition[DO_KWD]
+            )
+
+        try:
+            return self._resolve_stage(context, name, definition)
+        except ContextError as exc:
+            format_and_raise(exc, f"stage '{name}'", self.relpath)
 
     def resolve(self):
-        """Used for testing purposes, otherwise use resolve_one()."""
-        data = join(map(self.resolve_one, self.get_keys()))
-        logger.trace(  # type: ignore[attr-defined]
-            "Resolved dvc.yaml:\n%s", data
-        )
+        stages = self.data.get(STAGES_KWD, {})
+        data = join(starmap(self._resolve_entry, stages.items())) or {}
+        logger.trace("Resolved dvc.yaml:\n%s", data)
         return {STAGES_KWD: data}
 
-    def has_key(self, key: str):
-        return self._has_group_and_key(*split_foreach_name(key))
+    def _resolve_stage(self, context: Context, name: str, definition) -> dict:
+        definition = deepcopy(definition)
+        self.set_context_from(
+            context, definition.pop(SET_KWD, {}), source=[name, "set"]
+        )
+        wdir = self._resolve_wdir(context, name, definition.get(WDIR_KWD))
+        if self.wdir != wdir:
+            logger.debug(
+                "Stage %s has different wdir than dvc.yaml file", name
+            )
 
-    def _has_group_and_key(self, group: str, key: str = None):
+        vars_ = definition.pop(VARS_KWD, [])
+        # FIXME: Should `vars` be templatized?
         try:
-            definition = self.definitions[group]
-        except KeyError:
-            return False
+            self.load_from_vars(
+                context,
+                vars_,
+                wdir,
+                skip_imports=deepcopy(self.imported_files),
+                stage_name=name,
+            )
+        except VarsAlreadyLoaded as exc:
+            format_and_raise(exc, f"'stages.{name}.vars'", self.relpath)
 
-        if key:
-            return isinstance(
-                definition, ForeachDefinition
-            ) and definition.has_member(key)
-        return not isinstance(definition, ForeachDefinition)
+        logger.trace(  # type: ignore[attr-defined]
+            "Context during resolution of stage %s:\n%s", name, context
+        )
 
-    @collecting
-    def get_keys(self):
-        for name, definition in self.definitions.items():
-            if isinstance(definition, ForeachDefinition):
-                yield from definition.get_generated_names()
-                continue
-            yield name
+        with context.track():
+            resolved = {}
+            for key, value in definition.items():
+                # NOTE: we do not pop "wdir", and resolve it again
+                # this does not affect anything and is done to try to
+                # track the source of `wdir` interpolation.
+                # This works because of the side-effect that we do not
+                # allow overwriting and/or str interpolating complex objects.
+                # Fix if/when those assumptions are no longer valid.
+                try:
+                    resolved[key] = context.resolve(value)
+                except (ParseError, KeyNotInContext) as exc:
+                    format_and_raise(
+                        exc, f"'stages.{name}.{key}'", self.relpath
+                    )
 
-    def track_vars(self, name: str, vars_) -> None:
-        self.tracked_vars[name] = vars_
+        self.tracked_vars[name] = self._get_vars(context)
+        return {name: resolved}
 
-
-class EntryDefinition:
-    def __init__(
-        self,
-        resolver: DataResolver,
-        context: Context,
-        name: str,
-        definition: DictStr,
-        where: str = STAGES_KWD,
-    ):
-        self.resolver = resolver
-        self.wdir = self.resolver.wdir
-        self.relpath = self.resolver.relpath
-        self.context = context
-        self.name = name
-        self.definition = definition
-        self.where = where
+    def _get_vars(self, context: Context):
+        tracked = defaultdict(dict)
+        for path, vars_ in context.tracked.items():
+            for var in vars_:
+                tracked[path][var] = context.select(var, unwrap=True)
+        return tracked
 
     def _resolve_wdir(
         self, context: Context, name: str, wdir: str = None
@@ -237,221 +235,87 @@ class EntryDefinition:
             return self.wdir
 
         try:
-            wdir = to_str(context.resolve_str(wdir))
+            wdir = str(context.resolve_str(wdir, unwrap=True))
         except (ContextError, ParseError) as exc:
-            format_and_raise(exc, f"'{self.where}.{name}.wdir'", self.relpath)
-        return self.wdir / wdir
+            format_and_raise(exc, f"'stages.{name}.wdir'", self.relpath)
+        return self.wdir / str(wdir)
 
-    def resolve(self, **kwargs):
-        try:
-            return self.resolve_stage(**kwargs)
-        except ContextError as exc:
-            format_and_raise(exc, f"stage '{self.name}'", self.relpath)
-
-    def resolve_stage(self, skip_checks: bool = False) -> DictStr:
-        context = self.context
-        name = self.name
-        if not skip_checks:
-            # we can check for syntax errors as we go for interpolated entries,
-            # but for foreach-generated ones, once is enough, which it does
-            # that itself. See `ForeachDefinition.do_definition`.
-            check_syntax_errors(self.definition, name, self.relpath)
-
-        # we need to pop vars from generated/evaluated data
-        definition = deepcopy(self.definition)
-
-        wdir = self._resolve_wdir(context, name, definition.get(WDIR_KWD))
-        vars_ = definition.pop(VARS_KWD, [])
-        # FIXME: Should `vars` be templatized?
-        check_interpolations(vars_, f"{self.where}.{name}.vars", self.relpath)
-        if vars_:
-            # Optimization: Lookahead if it has any vars, if it does not, we
-            # don't need to clone them.
-            context = Context.clone(context)
-
-        try:
-            fs = self.resolver.fs
-            context.load_from_vars(fs, vars_, wdir, stage_name=name)
-        except VarsAlreadyLoaded as exc:
-            format_and_raise(exc, f"'{self.where}.{name}.vars'", self.relpath)
-
-        logger.trace(  # type: ignore[attr-defined]
-            "Context during resolution of stage %s:\n%s", name, context
+    def _foreach(self, context: Context, name: str, foreach_data, do_data):
+        iterable = self._resolve_foreach_data(context, name, foreach_data)
+        args = (context, name, do_data, iterable)
+        it = (
+            range(len(iterable))
+            if not isinstance(iterable, Mapping)
+            else iterable
         )
+        gen = (self._each_iter(*args, i) for i in it)
+        return join(gen)
 
-        with context.track() as tracked_data:
-            # NOTE: we do not pop "wdir", and resolve it again
-            # this does not affect anything and is done to try to
-            # track the source of `wdir` interpolation.
-            # This works because of the side-effect that we do not
-            # allow overwriting and/or str interpolating complex objects.
-            # Fix if/when those assumptions are no longer valid.
-            resolved = {
-                key: self._resolve(context, value, key, skip_checks)
-                for key, value in definition.items()
-            }
+    def _each_iter(self, context: Context, name: str, do_data, iterable, key):
+        value = iterable[key]
+        c = Context.clone(context)
+        suffix = c["item"] = value
+        if isinstance(iterable, Mapping):
+            suffix = c["key"] = key
 
-        self.resolver.track_vars(name, tracked_data)
-        return {name: resolved}
-
-    def _resolve(
-        self, context: "Context", value: Any, key: str, skip_checks: bool
-    ) -> DictStr:
+        generated = f"{name}{JOIN}{suffix}"
         try:
-            return context.resolve(
-                value, skip_interpolation_checks=skip_checks
-            )
-        except (ParseError, KeyNotInContext) as exc:
+            return self._resolve_stage(c, generated, do_data)
+        except ContextError as exc:
+            # pylint: disable=no-member
+            if isinstance(exc, MergeError) and exc.key in self._inserted_keys(
+                iterable
+            ):
+                raise ResolveError(
+                    f"attempted to redefine '{exc.key}' in stage '{generated}'"
+                    " generated through 'foreach'"
+                )
             format_and_raise(
-                exc, f"'{self.where}.{self.name}.{key}'", self.relpath
+                exc, f"stage '{generated}' (gen. from '{name}')", self.relpath
             )
 
-
-class IterationPair(NamedTuple):
-    key: str = "key"
-    value: str = "item"
-
-
-class ForeachDefinition:
-    def __init__(
-        self,
-        resolver: DataResolver,
-        context: Context,
-        name: str,
-        definition: DictStr,
-        where: str = STAGES_KWD,
+    def _resolve_foreach_data(
+        self, context: "Context", name: str, foreach_data
     ):
-        self.resolver = resolver
-        self.relpath = self.resolver.relpath
-        self.context = context
-        self.name = name
-
-        assert DO_KWD in definition
-        self.foreach_data = definition[FOREACH_KWD]
-        self._do_definition = definition[DO_KWD]
-
-        self.pair = IterationPair()
-        self.where = where
-
-    @cached_property
-    def do_definition(self):
-        # optimization: check for syntax errors only once for `foreach` stages
-        check_syntax_errors(self._do_definition, self.name, self.relpath)
-        return self._do_definition
-
-    @cached_property
-    def resolved_iterable(self):
-        return self._resolve_foreach_data()
-
-    def _resolve_foreach_data(self) -> SeqOrMap:
         try:
-            iterable = self.context.resolve(self.foreach_data, unwrap=False)
+            iterable = context.resolve(foreach_data, unwrap=False)
         except (ContextError, ParseError) as exc:
-            format_and_raise(
-                exc, f"'{self.where}.{self.name}.foreach'", self.relpath
-            )
-
-        # foreach data can be a resolved dictionary/list.
-        self._check_is_map_or_seq(iterable)
-        # foreach stages will have `item` and `key` added to the context
-        # so, we better warn them if they have them already in the context
-        # from the global vars. We could add them in `set_temporarily`, but
-        # that'd make it display for each iteration.
-        self._warn_if_overwriting(self._inserted_keys(iterable))
-        return iterable
-
-    def _check_is_map_or_seq(self, iterable):
-        if not is_map_or_seq(iterable):
-            node = iterable.value if isinstance(iterable, Node) else iterable
-            typ = type(node).__name__
+            format_and_raise(exc, f"'stages.{name}.foreach'", self.relpath)
+        if isinstance(iterable, str) or not isinstance(
+            iterable, (Sequence, Mapping)
+        ):
             raise ResolveError(
-                f"failed to resolve '{self.where}.{self.name}.foreach'"
-                f" in '{self.relpath}': expected list/dictionary, got " + typ
+                f"failed to resolve 'stages.{name}.foreach'"
+                f" in '{self.relpath}': expected list/dictionary, got "
+                + type(
+                    iterable.value if isinstance(iterable, Node) else iterable
+                ).__name__
             )
 
-    def _warn_if_overwriting(self, keys: List[str]):
-        warn_for = [k for k in keys if k in self.context]
+        warn_for = [k for k in self._inserted_keys(iterable) if k in context]
         if warn_for:
-            linking_verb = "is" if len(warn_for) == 1 else "are"
             logger.warning(
                 "%s %s already specified, "
                 "will be overwritten for stages generated from '%s'",
                 " and ".join(warn_for),
-                linking_verb,
-                self.name,
+                "is" if len(warn_for) == 1 else "are",
+                name,
             )
 
-    def _inserted_keys(self, iterable) -> List[str]:
-        keys = [self.pair.value]
+        return iterable
+
+    @staticmethod
+    def _inserted_keys(iterable):
+        keys = ["item"]
         if isinstance(iterable, Mapping):
-            keys.append(self.pair.key)
+            keys.append("key")
         return keys
 
-    @cached_property
-    def normalized_iterable(self):
-        """Convert sequence to Mapping with keys normalized."""
-        iterable = self.resolved_iterable
-        if isinstance(iterable, Mapping):
-            return iterable
-
-        assert isinstance(iterable, Sequence)
-        if any(map(is_map_or_seq, iterable)):
-            # if the list contains composite data, index are the keys
-            return {to_str(idx): value for idx, value in enumerate(iterable)}
-
-        # for simple lists, eg: ["foo", "bar"],  contents are the key itself
-        return {to_str(value): value for value in iterable}
-
-    def has_member(self, key: str) -> bool:
-        return key in self.normalized_iterable
-
-    def get_generated_names(self):
-        return list(map(self._generate_name, self.normalized_iterable))
-
-    def _generate_name(self, key: str) -> str:
-        return f"{self.name}{JOIN}{key}"
-
-    def resolve_all(self) -> DictStr:
-        return join(map(self.resolve_one, self.normalized_iterable))
-
-    def resolve_one(self, key: str) -> DictStr:
-        return self._each_iter(key)
-
-    def _each_iter(self, key: str) -> DictStr:
-        err_message = f"Could not find '{key}' in foreach group '{self.name}'"
-        with reraise(KeyError, EntryNotFound(err_message)):
-            value = self.normalized_iterable[key]
-
-        # NOTE: we need to use resolved iterable/foreach-data,
-        # not the normalized ones to figure out whether to make item/key
-        # available
-        inserted = self._inserted_keys(self.resolved_iterable)
-        temp_dict = {self.pair.value: value}
-        key_str = self.pair.key
-        if key_str in inserted:
-            temp_dict[key_str] = key
-
-        with self.context.set_temporarily(temp_dict, reserve=True):
-            # optimization: item and key can be removed on __exit__() as they
-            # are top-level values, and are not merged recursively.
-            # This helps us avoid cloning context, which is slower
-            # (increasing the size of the context might increase
-            # the no. of items to be generated which means more cloning,
-            # i.e. quadratic complexity).
-            generated = self._generate_name(key)
-            entry = EntryDefinition(
-                self.resolver, self.context, generated, self.do_definition
-            )
-            try:
-                # optimization: skip checking for syntax errors on each foreach
-                # generated stages. We do it once when accessing do_definition.
-                return entry.resolve_stage(skip_checks=True)
-            except ContextError as exc:
-                format_and_raise(
-                    exc, f"stage '{generated}'", self.relpath,
-                )
-
-            # let mypy know that this state is unreachable as format_and_raise
-            # does not return at all (it's not able to understand it for some
-            # reason)
-            raise AssertionError("unreachable")
+    @classmethod
+    def set_context_from(cls, context: Context, to_set, source=None):
+        try:
+            for key, value in to_set.items():
+                src_set = [*(source or []), key]
+                context.set(key, value, source=".".join(src_set))
+        except SetError as exc:
+            _reraise_err(ResolveError, str(exc), from_exc=exc)

dvc/parsing/context.py

@@ -1,4 +1,3 @@
-import logging
 import os
 from abc import ABC, abstractmethod
 from collections import defaultdict
@@ -6,58 +5,44 @@ from collections.abc import Mapping, MutableMapping, MutableSequence, Sequence
 from contextlib import contextmanager
 from copy import deepcopy
 from dataclasses import dataclass, field, replace
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, List, Optional, Union
 
-from funcy import identity, lfilter, nullcontext, select
+from funcy import identity, rpartial
 
 from dvc.exceptions import DvcException
 from dvc.parsing.interpolate import (
+    ParseError,
     get_expression,
     get_matches,
     is_exact_string,
-    normalize_key,
-    recurse,
+    is_interpolated_string,
     str_interpolate,
 )
 from dvc.path_info import PathInfo
 from dvc.utils import relpath
+from dvc.utils.serialize import LOADERS
 
-logger = logging.getLogger(__name__)
 SeqOrMap = Union[Sequence, Mapping]
-DictStr = Dict[str, Any]
 
 
 class ContextError(DvcException):
     pass
 
 
-class ReservedKeyError(ContextError):
-    def __init__(self, keys, path=None):
-        from dvc.utils.humanize import join
-
-        self.keys = keys
-        self.path = path
-
-        n = "key" + ("s" if len(keys) > 1 else "")
-        msg = f"attempted to modify reserved {n} {join(keys)}"
-        if path:
-            msg += f" in '{path}'"
-        super().__init__(msg)
+class SetError(ContextError):
+    pass
 
 
 class MergeError(ContextError):
     def __init__(self, key, new, into):
         self.key = key
-        to_node = into[key]
-        if not isinstance(to_node, Node) or not isinstance(new, Node):
+        if not isinstance(into[key], Node) or not isinstance(new, Node):
             super().__init__(
                 f"cannot merge '{key}' as it already exists in {into}"
             )
             return
 
-        assert isinstance(to_node, Node)
-        assert isinstance(new, Node)
-        preexisting = to_node.meta.source
+        preexisting = into[key].meta.source
         new_src = new.meta.source
         path = new.meta.path()
         super().__init__(
@@ -66,22 +51,15 @@ class MergeError(ContextError):
         )
 
 
-class ParamsLoadError(ContextError):
+class ParamsFileNotFound(ContextError):
     pass
 
 
-class KeyNotInContext(ContextError, KeyError):
-    def __init__(self, key: str) -> None:
-        self.key: str = key
+class KeyNotInContext(ContextError):
+    def __init__(self, key) -> None:
+        self.key = key
         super().__init__(f"Could not find '{key}'")
 
-    def __str__(self):
-        return self.msg
-
-
-class VarsAlreadyLoaded(ContextError):
-    pass
-
 
 def _merge(into, update, overwrite):
     for key, val in update.items():
@@ -94,13 +72,6 @@ def _merge(into, update, overwrite):
             assert isinstance(into[key], Node)
 
 
-def recurse_not_a_node(data: dict):
-    def func(item):
-        assert not isinstance(item, Node)
-
-    return recurse(func)(data)
-
-
 @dataclass
 class Meta:
     source: Optional[str] = None
@@ -121,13 +92,11 @@ class Meta:
         return ".".join(self.dpaths)
 
 
-def _default_meta() -> Meta:
-    return Meta()
+def _default_meta():
+    return Meta(source=None)
 
 
 class Node:
-    meta: Meta
-
     def get_sources(self):
         raise NotImplementedError
 
@@ -178,17 +147,18 @@ class Container(Node, ABC):
         if value is None or isinstance(value, PRIMITIVES):
             assert meta
             return Value(value, meta=meta)
-        if isinstance(value, Node):
+        elif isinstance(value, Node):
             return value
-        if isinstance(value, (list, dict)):
+        elif isinstance(value, (list, dict)):
             assert meta
             container = CtxDict if isinstance(value, dict) else CtxList
             return container(value, meta=meta)
-        msg = (
-            "Unsupported value of type "
-            f"'{type(value).__name__}' in '{meta}'"
-        )
-        raise TypeError(msg)
+        else:
+            msg = (
+                "Unsupported value of type "
+                f"'{type(value).__name__}' in '{meta}'"
+            )
+            raise TypeError(msg)
 
     def __repr__(self):
         return repr(self.data)
@@ -257,12 +227,6 @@ class CtxList(Container, MutableSequence):
     def value(self):
         return [node.value for node in self]
 
-    def __deepcopy__(self, _):
-        # optimization: we don't support overriding a list
-        new = CtxList([])
-        new.data = self.data[:]  # Short-circuiting __setitem__
-        return new
-
 
 class CtxDict(Container, MutableMapping):
     def __init__(self, mapping: Mapping = None, meta: Meta = None, **kwargs):
@@ -280,21 +244,14 @@ class CtxDict(Container, MutableMapping):
             return
         return super().__setitem__(key, value)
 
-    def merge_update(self, other, overwrite=False):
-        _merge(self, other, overwrite=overwrite)
+    def merge_update(self, *args, overwrite=False):
+        for d in args:
+            _merge(self, d, overwrite=overwrite)
 
     @property
     def value(self):
         return {key: node.value for key, node in self.items()}
 
-    def __deepcopy__(self, _):
-        new = CtxDict()
-        for k, v in self.items():
-            new.data[k] = (
-                deepcopy(v) if isinstance(v, Container) else v
-            )  # short-circuiting __setitem__
-        return new
-
 
 class Context(CtxDict):
     def __init__(self, *args, **kwargs):
@@ -303,23 +260,18 @@ class Context(CtxDict):
         """
         super().__init__(*args, **kwargs)
         self._track = False
-        self._tracked_data = defaultdict(dict)
-        self.imports = {}
-        self._reserved_keys = {}
+        self._tracked_data = defaultdict(set)
 
     @contextmanager
     def track(self):
         self._track = True
-        yield self._tracked_data
-
+        yield
         self._track = False
-        self._tracked_data = defaultdict(dict)
 
     def _track_data(self, node):
-        if not self._track or not isinstance(node, Node):
+        if not self._track:
             return
 
-        assert isinstance(node, Node)
         if node.meta and node.meta.local:
             return
 
@@ -328,10 +280,14 @@ class Context(CtxDict):
                 continue
             params_file = self._tracked_data[source]
             keys = [keys] if isinstance(keys, str) else keys
-            params_file.update({key: node.value for key in keys})
+            params_file.update(keys)
+
+    @property
+    def tracked(self):
+        return self._tracked_data
 
     def select(
-        self, key: str, unwrap: bool = False
+        self, key: str, unwrap=False
     ):  # pylint: disable=arguments-differ
         """Select the item using key, similar to `__getitem__`
            but can track the usage of the data on interpolation
@@ -344,195 +300,101 @@ class Context(CtxDict):
                     Defaults to False. Note that the default is different from
                     `resolve`.
         """
-        normalized = normalize_key(key)
         try:
-            node = super().select(normalized)
+            node = super().select(key)
         except ValueError as exc:
             raise KeyNotInContext(key) from exc
 
-        assert isinstance(node, Node)
         self._track_data(node)
+        assert isinstance(node, Node)
         return node.value if unwrap else node
 
     @classmethod
-    def load_from(
-        cls, fs, path: PathInfo, select_keys: List[str] = None
-    ) -> "Context":
-        from dvc.utils.serialize import LOADERS
-
+    def load_from(cls, tree, path: PathInfo, select_keys=None) -> "Context":
         file = relpath(path)
-        if not fs.exists(path):
-            raise ParamsLoadError(f"'{file}' does not exist")
-        if fs.isdir(path):
-            raise ParamsLoadError(f"'{file}' is a directory")
+        if not tree.exists(path):
+            raise ParamsFileNotFound(f"'{file}' does not exist")
 
         _, ext = os.path.splitext(file)
         loader = LOADERS[ext]
 
-        data = loader(path, fs=fs)
-        if not isinstance(data, Mapping):
-            typ = type(data).__name__
-            raise ParamsLoadError(
-                f"expected a dictionary, got '{typ}' in file '{file}'"
-            )
-
+        data = loader(path, tree=tree)
+        select_keys = select_keys or []
         if select_keys:
             try:
                 data = {key: data[key] for key in select_keys}
             except KeyError as exc:
                 key, *_ = exc.args
-                raise ParamsLoadError(
+                raise ContextError(
                     f"could not find '{key}' in '{file}'"
                 ) from exc
 
         meta = Meta(source=file, local=False)
-        ctx = cls(data, meta=meta)
-        ctx.imports[os.path.abspath(path)] = select_keys
-        return ctx
-
-    def merge_update(self, other: "Context", overwrite=False):
-        matches = select(lambda key: key in other, self._reserved_keys.keys())
-        if matches:
-            raise ReservedKeyError(matches)
-        return super().merge_update(other, overwrite=overwrite)
+        return cls(data, meta=meta)
 
     def merge_from(
-        self, fs, item: str, wdir: PathInfo, overwrite=False,
+        self, tree, path: PathInfo, overwrite=False, select_keys=None,
     ):
-        path, _, keys_str = item.partition(":")
-        select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
-        path_info = wdir / path
-
-        abspath = os.path.abspath(path_info)
-        if abspath in self.imports:
-            if not select_keys and self.imports[abspath] is None:
-                return  # allow specifying complete filepath multiple times
-            self.check_loaded(abspath, item, select_keys)
-
-        ctx = Context.load_from(fs, path_info, select_keys)
-
-        try:
-            self.merge_update(ctx, overwrite=overwrite)
-        except ReservedKeyError as exc:
-            raise ReservedKeyError(exc.keys, item) from exc
-
-        cp = ctx.imports[abspath]
-        if abspath not in self.imports:
-            self.imports[abspath] = cp
-        elif cp:
-            self.imports[abspath].extend(cp)
-
-    def check_loaded(self, path, item, keys):
-        if not keys and isinstance(self.imports[path], list):
-            raise VarsAlreadyLoaded(
-                f"cannot load '{item}' as it's partially loaded already"
-            )
-        elif keys and self.imports[path] is None:
-            raise VarsAlreadyLoaded(
-                f"cannot partially load '{item}' as it's already loaded."
-            )
-        elif isinstance(self.imports[path], list):
-            if not set(keys).isdisjoint(set(self.imports[path])):
-                raise VarsAlreadyLoaded(
-                    f"cannot load '{item}' as it's partially loaded already"
-                )
-
-    def load_from_vars(
-        self,
-        fs,
-        vars_: List,
-        wdir: PathInfo,
-        stage_name: str = None,
-        default: str = None,
-    ):
-        if default:
-            to_import = wdir / default
-            if fs.exists(to_import):
-                self.merge_from(fs, default, wdir)
-            else:
-                msg = "%s does not exist, it won't be used in parametrization"
-                logger.trace(msg, to_import)  # type: ignore[attr-defined]
-
-        stage_name = stage_name or ""
-        for index, item in enumerate(vars_):
-            assert isinstance(item, (str, dict))
-            if isinstance(item, str):
-                self.merge_from(fs, item, wdir)
-            else:
-                joiner = "." if stage_name else ""
-                meta = Meta(source=f"{stage_name}{joiner}vars[{index}]")
-                self.merge_update(Context(item, meta=meta))
-
-    def __deepcopy__(self, _):
-        new = Context(super().__deepcopy__(_))
-        new.meta = deepcopy(self.meta)
-        new.imports = deepcopy(self.imports)
-        new._reserved_keys = deepcopy(self._reserved_keys)
-        return new
+        self.merge_update(
+            Context.load_from(tree, path, select_keys), overwrite=overwrite
+        )
 
     @classmethod
     def clone(cls, ctx: "Context") -> "Context":
         """Clones given context."""
-        return deepcopy(ctx)
+        return cls(deepcopy(ctx.data))
 
-    @contextmanager
-    def reserved(self, *keys: str):
-        """Allow reserving some keys so that they cannot be overwritten.
+    def set(self, key, value, source=None):
+        """
+        Sets a value, either non-interpolated values to a key,
+        or an interpolated string after resolving it.
 
-        Ideally, we should delegate this to a separate container
-        and support proper namespacing so that we could support `env` features.
-        But for now, just `item` and `key`, this should do.
+        >>> c = Context({"foo": "foo", "bar": [1, 2], "lorem": {"a": "z"}})
+        >>> c.set("foobar", "${bar}")
+        >>> c
+        {'foo': 'foo', 'bar': [1, 2], 'lorem': {'a': 'z'}, 'foobar': [1, 2]}
         """
-        # using dict to make the error messages ordered
-        new = dict.fromkeys(
-            [key for key in keys if key not in self._reserved_keys]
-        )
-        self._reserved_keys.update(new)
         try:
-            yield
-        finally:
-            for key in new.keys():
-                self._reserved_keys.pop(key)
+            self._set(key, value, source)
+        except (ParseError, ValueError, ContextError) as exc:
+            sp = "\n" if isinstance(exc, ParseError) else " "
+            raise SetError(f"Failed to set '{source}':{sp}{str(exc)}") from exc
 
-    @contextmanager
-    def set_temporarily(self, to_set: DictStr, reserve: bool = False):
-        cm = self.reserved(*to_set) if reserve else nullcontext()
+    def _set(self, key, value, source):
+        if key in self:
+            raise ValueError(f"Cannot set '{key}', key already exists")
+        if isinstance(value, str):
+            self._check_joined_with_interpolation(key, value)
+            value = self.resolve_str(value, unwrap=False)
+        elif isinstance(value, (Sequence, Mapping)):
+            self._check_not_nested_collection(key, value)
+            self._check_interpolation_collection(key, value)
+        self[key] = self._convert_with_meta(value, Meta(source=source))
 
-        non_existing = frozenset(to_set.keys() - self.keys())
-        prev = {key: self[key] for key in to_set if key not in non_existing}
-        temp = CtxDict(to_set)
-        self.update(temp)
-
-        try:
-            with cm:
-                yield
-        finally:
-            self.update(prev)
-            for key in non_existing:
-                self.data.pop(key, None)
-
-    def resolve(
-        self, src, unwrap=True, skip_interpolation_checks=False
-    ) -> Any:
+    def resolve(self, src, unwrap=True):
         """Recursively resolves interpolation and returns resolved data.
 
         Args:
             src: Data (str/list/dict etc.) to resolve
             unwrap: Unwrap CtxDict/CtxList/Value to it's original data if
-                inside `src`. Defaults to True.
-            skip_interpolation_checks: Skip interpolation checks for error
-                The callee is responsible to check for errors in advance.
+                    inside `src`. Defaults to True.
 
         >>> c = Context({"three": 3})
         >>> c.resolve({"lst": [1, 2, "${three}"]})
         {'lst': [1, 2, 3]}
         """
-        func = recurse(self.resolve_str)
-        return func(src, unwrap, skip_interpolation_checks)
+        Seq = (list, tuple, set)
 
-    def resolve_str(
-        self, src: str, unwrap=True, skip_interpolation_checks=False
-    ) -> str:
+        resolve = rpartial(self.resolve, unwrap)
+        if isinstance(src, Mapping):
+            return dict(map(resolve, kv) for kv in src.items())
+        elif isinstance(src, Seq):
+            return type(src)(map(resolve, src))
+        elif isinstance(src, str):
+            return self.resolve_str(src, unwrap=unwrap)
+        return src
+
+    def resolve_str(self, src: str, unwrap=True):
         """Resolves interpolated string to it's original value,
         or in case of multiple interpolations, a combined string.
 
@@ -540,20 +402,47 @@ class Context(CtxDict):
         >>> c.resolve_str("${enabled}")
         True
         >>> c.resolve_str("enabled? ${enabled}")
-        'enabled? true'
+        'enabled? True'
         """
         matches = get_matches(src)
         if is_exact_string(src, matches):
             # replace "${enabled}", if `enabled` is a boolean, with it's actual
             # value rather than it's string counterparts.
-            expr = get_expression(
-                matches[0], skip_checks=skip_interpolation_checks
-            )
+            expr = get_expression(matches[0])
             return self.select(expr, unwrap=unwrap)
         # but not "${num} days"
-        return str_interpolate(
-            src, matches, self, skip_checks=skip_interpolation_checks
+        return str_interpolate(src, matches, self)
+
+    @staticmethod
+    def _check_not_nested_collection(key: str, value: SeqOrMap):
+        values = value.values() if isinstance(value, Mapping) else value
+        has_nested = any(
+            not isinstance(item, str) and isinstance(item, (Mapping, Sequence))
+            for item in values
         )
+        if has_nested:
+            raise ValueError(f"Cannot set '{key}', has nested dict/list")
+
+    @staticmethod
+    def _check_interpolation_collection(key: str, value: SeqOrMap):
+        values = value.values() if isinstance(value, Mapping) else value
+        interpolated = any(is_interpolated_string(item) for item in values)
+        if interpolated:
+            raise ValueError(
+                f"Cannot set '{key}', "
+                "having interpolation inside "
+                f"'{type(value).__name__}' is not supported."
+            )
+
+    @staticmethod
+    def _check_joined_with_interpolation(key: str, value: str):
+        matches = get_matches(value)
+        if matches and not is_exact_string(value, matches):
+            raise ValueError(
+                f"Cannot set '{key}', "
+                "joining string with interpolated string "
+                "is not supported"
+            )
 
 
 if __name__ == "__main__":