gha: don't upload pkg artifacts

* build: add osx signing/notarization

* gha: temporarily enable pkg builds

.github/ISSUE_TEMPLATE/bug_report.md

@@ -48,14 +48,17 @@ A clear and concise description of what you expect to happen.
 This is required to ensure that we can reproduce the bug.
 -->
 
-**Output of `dvc version`:**
+**Output of `dvc doctor`:**
 
 ```console
-$ dvc version
+$ dvc doctor
 ```
 
 **Additional Information (if any):**
 
 <!--
+Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
+
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
--->
+If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
+-->

.github/workflows/packages.yaml

@@ -6,20 +6,25 @@ on:
   release:
     types: [released, prereleased]
 jobs:
-  linux:
+  deb:
     runs-on: ubuntu-16.04
     steps:
     - uses: actions/[email protected]
-    - name: Set up Python 3.7
+    - name: Set up Python 3.8
       uses: actions/[email protected]
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build deb
+      run: python scripts/build.py deb
     - name: Publish deb
       if: github.event_name == 'release'
       uses: actions/[email protected]
@@ -27,9 +32,28 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_content_type: binary/octet-stream
+  rpm:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build rpm
+      run: python scripts/build.py rpm
     - name: Publish rpm
       if: github.event_name == 'release'
       uses: actions/[email protected]
@@ -37,23 +61,31 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
         asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
         asset_content_type: binary/octet-stream
-  mac:
+  osxpkg:
     runs-on: macos-10.15
     steps:
     - uses: actions/[email protected]
-    - name: Set up Python 3.7
+    - name: Set up Python 3.8
       uses: actions/[email protected]
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
+    - name: Install deps
+      run: |
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
     - name: Build
-      run: ./scripts/build_posix.sh
+      run: python scripts/build.py osxpkg
     - name: Publish
       if: github.event_name == 'release'
       uses: actions/[email protected]
@@ -61,19 +93,26 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
         asset_name: dvc-${{ github.event.release.tag_name }}.pkg
         asset_content_type: binary/octet-stream
-  windows:
+  exe:
     runs-on: windows-2019
     steps:
     - uses: actions/[email protected]
-    - name: Set up Python 3.7
+    - name: Set up Python 3.8
       uses: actions/[email protected]
       with:
-        python-version: 3.7
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+    - name: Pull images
+      run: |
+        dvc pull
     - name: Build
-      run: scripts\build_windows.cmd
+      run: python scripts\build.py exe
       shell: cmd
     - name: Publish
       if: github.event_name == 'release'
@@ -82,10 +121,10 @@ jobs:
       uses: actions/[email protected]
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
         asset_name: dvc-${{ github.event.release.tag_name }}.exe
         asset_content_type: binary/octet-stream
-  python:
+  pip:
     runs-on: ubuntu-16.04
     steps:
     - uses: actions/[email protected]
@@ -110,11 +149,11 @@ jobs:
     - name: Build the package
       uses: snapcore/action-build@v1
       id: snapcraft
-    - name: Test the package
-      run: |
-        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
-        dvc version
-        sudo snap remove dvc
+#    - name: Test the package
+#      run: |
+#        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
+#        dvc version
+#        sudo snap remove dvc
     - name: Publish to the edge channel
       uses: snapcore/action-publish@v1
       if: github.event_name == 'push'

.github/workflows/tests.yaml

@@ -94,14 +94,3 @@ jobs:
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
-    - name: Post TAP
-      # Running custom fork with an option, that does not fail CI on error
-      uses: skshetry/flaky-service/packages/action@master
-      if: always() && github.ref == 'refs/heads/master'
-      with:
-        file-path: ${{github.workspace}}/testresults.tap
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        os: ${{runner.os}}
-        matrix: ${{toJson(matrix)}}
-        repo-description: ${{github.event.repository.description}}
-        fail_ci_if_error: false

.pylintrc

@@ -15,10 +15,13 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook="import sys; \
-           from pathlib import Path; \
-           from pylint.config import find_pylintrc; \
-           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+init-hook="import sys;
+           from pathlib import Path;
+           from pylint.config import find_pylintrc;
+
+           tests_folder = Path(find_pylintrc()).parent / 'tests'
+           sys.path.append(str(tests_folder));
+           "
 
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
@@ -178,7 +181,10 @@ disable=format,
         # we use it to speedup/optimize
         import-outside-toplevel,
         fixme,
-
+        # we use cause of the exception for error message output
+        # so some of the times we don't use `raise from` to not show the cause
+        # of the error
+        raise-missing-from,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

README.rst

@@ -222,6 +222,9 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
+Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA.
+
+
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png
    :target: https://dvc.org
    :alt: DVC logo

dvc/api.py

@@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM
 from funcy import reraise
 
 from dvc.exceptions import OutputNotFoundError, PathMissingError
-from dvc.objects.stage import get_hash
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
@@ -29,8 +28,8 @@ def get_url(path, repo=None, rev=None, remote=None):
             raise OutputNotFoundError(path, repo)
 
         cloud = metadata.repo.cloud
-        hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb)
-        return cloud.get_url_for(remote, checksum=hash_info.value)
+        md5 = metadata.repo.dvcfs.info(path_info)["md5"]
+        return cloud.get_url_for(remote, checksum=md5)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/checkout.py

@@ -9,8 +9,9 @@ from dvc.exceptions import (
     ConfirmRemoveError,
     DvcException,
 )
-from dvc.objects import ObjectFormatError, check, load
-from dvc.objects.stage import get_hash
+from dvc.objects import check, load
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.stage import stage
 from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
     slow_link_guard,
 )
@@ -30,7 +31,7 @@ def _changed(path_info, fs, obj, cache):
         return True
 
     try:
-        actual = get_hash(path_info, fs, obj.hash_info.name, cache)
+        actual = stage(cache, path_info, fs, obj.hash_info.name).hash_info
     except FileNotFoundError:
         logger.debug("'%s' doesn't exist.", path_info)
         return True
@@ -56,7 +57,7 @@ def _remove(path_info, fs, cache, force=False):
         fs.remove(path_info)
         return
 
-    current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache)
+    current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM).hash_info
     try:
         obj = load(cache, current)
         check(cache, obj)
@@ -174,10 +175,10 @@ def _checkout_file(
     return modified
 
 
-def _remove_redundant_files(path_info, fs, dir_info, cache, force):
+def _remove_redundant_files(path_info, fs, obj, cache, force):
     existing_files = set(fs.walk_files(path_info))
 
-    needed_files = {path_info.joinpath(*key) for key, _ in dir_info.items()}
+    needed_files = {path_info.joinpath(*key) for key, _ in obj}
     redundant_files = existing_files - needed_files
     for path in redundant_files:
         _remove(path, fs, cache, force)
@@ -197,11 +198,11 @@ def _checkout_dir(
 
     logger.debug("Linking directory '%s'.", path_info)
 
-    for entry_key, entry_hash_info in obj.hash_info.dir_info.items():
+    for entry_key, entry_obj in obj:
         entry_modified = _checkout_file(
             path_info.joinpath(*entry_key),
             fs,
-            cache.get(entry_hash_info),
+            entry_obj,
             cache,
             force,
             progress_callback,
@@ -211,10 +212,7 @@ def _checkout_dir(
             modified = True
 
     modified = (
-        _remove_redundant_files(
-            path_info, fs, obj.hash_info.dir_info, cache, force
-        )
-        or modified
+        _remove_redundant_files(path_info, fs, obj, cache, force) or modified
     )
 
     fs.repo.state.save(path_info, fs, obj.hash_info)

dvc/command/experiments.py

@@ -514,6 +514,11 @@ class CmdExperimentsRun(CmdRepro):
     def run(self):
         from dvc.command.metrics import _show_metrics
 
+        if self.args.reset and self.args.checkpoint_resume:
+            raise InvalidArgumentError(
+                "--reset and --rev are mutually exclusive."
+            )
+
         if self.args.reset:
             logger.info("Any existing checkpoints will be reset and re-run.")
 

dvc/command/plots.py

@@ -1,6 +1,5 @@
 import argparse
 import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
@@ -23,6 +22,8 @@ class CmdPlots(CmdBase):
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
+        from pathlib import Path
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -41,17 +42,24 @@ class CmdPlots(CmdBase):
                 logger.info(plots[target])
                 return 0
 
-            path = self.args.out or "plots.html"
-            path = os.path.join(os.getcwd(), path)
-
+            rel: str = self.args.out or "plots.html"
+            path = (Path.cwd() / rel).resolve()
             write(path, plots)
-
-            logger.info(f"file://{path}")
-
         except DvcException:
             logger.exception("")
             return 1
 
+        assert path.is_absolute()  # as_uri throws ValueError if not absolute
+        url = path.as_uri()
+        logger.info(url)
+        if self.args.open:
+            import webbrowser
+
+            opened = webbrowser.open(rel)
+            if not opened:
+                logger.error("Failed to open. Please try opening it manually.")
+                return 1
+
         return 0
 
 
@@ -229,3 +237,9 @@ def _add_output_arguments(parser):
         default=False,
         help="Show output in Vega format.",
     )
+    parser.add_argument(
+        "--open",
+        action="store_true",
+        default=False,
+        help="Open plot file directly in the browser.",
+    )

dvc/command/stage.py

@@ -255,7 +255,8 @@ def _add_common_args(parser):
         "--checkpoints",
         action="append",
         default=[],
-        help=argparse.SUPPRESS,
+        help="Declare checkpoint output file or directory for 'dvc exp run'. "
+        "Not compatible with 'dvc repro'.",
         metavar="<filename>",
     ).complete = completion.FILE
     parser.add_argument(

dvc/command/status.py

@@ -44,6 +44,8 @@ class CmdDataStatus(CmdDataBase):
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.ui import ui
+
         indent = 1 if self.args.cloud else 0
         try:
             st = self.repo.status(
@@ -68,14 +70,14 @@ class CmdDataStatus(CmdDataBase):
             elif st:
                 self._show(st, indent)
             elif not self.repo.stages:
-                logger.info(self.EMPTY_PROJECT_MSG)
+                ui.write(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
                     "remote"
                 )
-                logger.info(self.IN_SYNC_MSG.format(remote=remote))
+                ui.write(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                logger.info(self.UP_TO_DATE_MSG)
+                ui.write(self.UP_TO_DATE_MSG)
 
         except DvcException:
             logger.exception("")

dvc/dependency/repo.py

@@ -55,6 +55,7 @@ class RepoDependency(LocalDependency):
                 self.repo.odb.local,
                 path_info,
                 repo.repo_fs,
+                self.repo.odb.local.fs.PARAM_CHECKSUM,
                 follow_subrepos=False,
             ).hash_info
 
@@ -78,6 +79,8 @@ class RepoDependency(LocalDependency):
 
     def download(self, to, jobs=None):
         from dvc.checkout import checkout
+        from dvc.config import NoRemoteError
+        from dvc.exceptions import NoOutputOrStageError
         from dvc.objects import save
         from dvc.objects.stage import stage
 
@@ -87,8 +90,17 @@ class RepoDependency(LocalDependency):
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
                 self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
             path_info = PathInfo(repo.root_dir) / self.def_path
+            try:
+                repo.fetch([path_info.fspath], jobs=jobs, recursive=True)
+            except (NoOutputOrStageError, NoRemoteError):
+                pass
             obj = stage(
-                odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False,
+                odb,
+                path_info,
+                repo.repo_fs,
+                odb.fs.PARAM_CHECKSUM,
+                jobs=jobs,
+                follow_subrepos=False,
             )
             save(odb, obj, jobs=jobs)
 

dvc/dir_info.py

@@ -1,117 +0,0 @@
-import posixpath
-from operator import itemgetter
-
-from funcy import cached_property
-from pygtrie import Trie
-
-from .hash_info import HashInfo
-
-
-def _diff(ancestor, other, allow_removed=False):
-    from dictdiffer import diff
-
-    from dvc.exceptions import MergeError
-
-    allowed = ["add"]
-    if allow_removed:
-        allowed.append("remove")
-
-    result = list(diff(ancestor, other))
-    for typ, _, _ in result:
-        if typ not in allowed:
-            raise MergeError(
-                "unable to auto-merge directories with diff that contains "
-                f"'{typ}'ed files"
-            )
-    return result
-
-
-def _merge(ancestor, our, their):
-    import copy
-
-    from dictdiffer import patch
-
-    our_diff = _diff(ancestor, our)
-    if not our_diff:
-        return copy.deepcopy(their)
-
-    their_diff = _diff(ancestor, their)
-    if not their_diff:
-        return copy.deepcopy(our)
-
-    # make sure there are no conflicting files
-    _diff(our, their, allow_removed=True)
-
-    return patch(our_diff + their_diff, ancestor)
-
-
-class DirInfo:
-    PARAM_RELPATH = "relpath"
-
-    def __init__(self):
-        self._dict = {}
-
-    def as_dict(self):
-        return self._dict.copy()
-
-    def add(self, key, hash_info):
-        self.__dict__.pop("trie", None)
-        self._dict[key] = hash_info
-
-    def get(self, key, default=None):
-        return self._dict.get(key, default)
-
-    @cached_property
-    def trie(self):
-        return Trie(self._dict)
-
-    @property
-    def size(self):
-        try:
-            return sum(hash_info.size for _, hash_info in self._dict.items())
-        except TypeError:
-            return None
-
-    @property
-    def nfiles(self):
-        return len(self._dict)
-
-    def items(self):
-        for key, hash_info in self._dict.items():
-            yield key, hash_info
-
-    @classmethod
-    def from_list(cls, lst):
-        ret = DirInfo()
-        for _entry in lst:
-            entry = _entry.copy()
-            relpath = entry.pop(cls.PARAM_RELPATH)
-            parts = tuple(relpath.split(posixpath.sep))
-            hash_info = HashInfo.from_dict(entry)
-            ret.add(parts, hash_info)
-        return ret
-
-    def to_list(self):
-        # Sorting the list by path to ensure reproducibility
-        return sorted(
-            (
-                {
-                    # NOTE: not using hash_info.to_dict() because we don't want
-                    # size/nfiles fields at this point.
-                    hash_info.name: hash_info.value,
-                    self.PARAM_RELPATH: posixpath.sep.join(parts),
-                }
-                for parts, hash_info in self._dict.items()  # noqa: B301
-            ),
-            key=itemgetter(self.PARAM_RELPATH),
-        )
-
-    def merge(self, ancestor, their):
-        merged_dict = _merge(
-            ancestor.as_dict(), self.as_dict(), their.as_dict(),
-        )
-
-        merged = DirInfo()
-        for key, hi in merged_dict.items():
-            merged.add(key, hi)
-        return merged

dvc/fs/azure.py

@@ -48,8 +48,6 @@ class AzureFileSystem(FSSpecWrapper):
         self.path_info = self.PATH_CLS(url)
         self.bucket = self.path_info.bucket
 
-        self.login_method, self.login_info = self._prepare_credentials(config)
-
     def _prepare_credentials(self, config):
         from azure.identity.aio import DefaultAzureCredential
 
@@ -90,7 +88,8 @@ class AzureFileSystem(FSSpecWrapper):
         else:
             login_method = None
 
-        return login_method, login_info
+        self.login_method = login_method
+        return login_info
 
     @cached_property
     def _az_config(self):
@@ -111,7 +110,7 @@ class AzureFileSystem(FSSpecWrapper):
         from azure.core.exceptions import AzureError
 
         try:
-            file_system = AzureBlobFileSystem(**self.login_info)
+            file_system = AzureBlobFileSystem(**self.fs_args)
             if self.bucket not in [
                 container.rstrip("/") for container in file_system.ls("/")
             ]:

dvc/fs/base.py

@@ -283,6 +283,9 @@ class BaseFileSystem:
         self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
     ):
         from_infos = list(self.walk_files(from_info, **kwargs))
+        if not from_infos:
+            makedirs(to_info, exist_ok=True)
+            return None
         to_infos = (
             to_info / info.relative_to(from_info) for info in from_infos
         )

dvc/fs/dvc.py

@@ -48,9 +48,9 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # NOTE: use string paths here for performance reasons
         key = tuple(relpath(path_info, out.path_info).split(os.sep))
         out.get_dir_cache(remote=remote)
-        file_hash = out.hash_info.dir_info.get(key)
-        if file_hash:
-            return file_hash
+        obj = out.obj.trie.get(key)
+        if obj:
+            return obj.hash_info
         raise FileNotFoundError
 
     def open(  # type: ignore
@@ -131,21 +131,14 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # pull dir cache if needed
         out.get_dir_cache(**kwargs)
 
-        dir_cache = out.dir_cache
-        if not dir_cache:
-            raise FileNotFoundError
-
-        from dvc.objects import Tree
-
-        hash_info = Tree.save_dir_info(out.odb, dir_cache)
-        if hash_info != out.hash_info:
+        if not out.obj:
             raise FileNotFoundError
 
     def _add_dir(self, trie, out, **kwargs):
         self._fetch_dir(out, **kwargs)
 
         base = out.path_info.parts
-        for key, _ in out.dir_cache.items():  # noqa: B301
+        for key, _ in out.obj:  # noqa: B301
             trie[base + key] = None
 
     def _walk(self, root, trie, topdown=True, **kwargs):
@@ -241,9 +234,9 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         elif meta.part_of_output:
             (out,) = meta.outs
             key = path_info.relative_to(out.path_info).parts
-            hash_info = out.hash_info.dir_info.trie.get(key)
-            if hash_info:
-                ret["size"] = hash_info.size
-                ret[hash_info.name] = hash_info.value
+            obj = out.obj.trie.get(key)
+            if obj:
+                ret["size"] = obj.size
+                ret[obj.hash_info.name] = obj.hash_info.value
 
         return ret

dvc/fs/fsspec_wrapper.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+from functools import lru_cache
 
 from dvc.progress import Tqdm
 
@@ -8,17 +9,29 @@ from .base import BaseFileSystem
 
 # pylint: disable=no-member
 class FSSpecWrapper(BaseFileSystem):
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+        self.fs_args = {"skip_instance_cache": True}
+        self.fs_args.update(self._prepare_credentials(config))
+
     def fs(self):
         raise NotImplementedError
 
+    @lru_cache(512)
     def _with_bucket(self, path):
         if isinstance(path, self.PATH_CLS):
             return f"{path.bucket}/{path.path}"
         return path
 
     def _strip_bucket(self, entry):
-        _, entry = entry.split("/", 1)
-        return entry
+        try:
+            bucket, path = entry.split("/", 1)
+        except ValueError:
+            # If there is no path attached, only returns
+            # the bucket (top-level).
+            bucket, path = entry, None
+
+        return path or bucket
 
     def _strip_buckets(self, entries, detail, prefix=None):
         for entry in entries:
@@ -36,11 +49,38 @@ class FSSpecWrapper(BaseFileSystem):
         entries within info() and ls(detail=True) calls"""
         return entry
 
+    def _prepare_credentials(self, config):  # pylint: disable=unused-argument
+        """Prepare the arguments for authentication to the
+        host filesystem"""
+        return {}
+
     def isdir(self, path_info):
-        return self.fs.isdir(self._with_bucket(path_info))
+        if not self.exists(path_info):
+            return False
+
+        # Directory in object storages are interpreted differently
+        # among different fsspec providers, so this logic is a temporary
+        # measure for us to adapt as of now. It checks whether it is a
+        # directory (as in a prefix with contents) or whether it is an empty
+        # file where it's name ends with a forward slash
+        entry = self.info(path_info)
+        return entry["type"] == "directory" or (
+            entry["size"] == 0
+            and entry["type"] == "file"
+            and entry["name"].endswith("/")
+        )
 
     def isfile(self, path_info):
-        return self.fs.isfile(self._with_bucket(path_info))
+        if not self.exists(path_info):
+            return False
+
+        return not self.isdir(path_info)
+
+    def is_empty(self, path_info):
+        entry = self.info(path_info)
+        if entry["type"] == "directory":
+            return not self.fs.ls(self._with_bucket(path_info))
+        return entry["size"] == 0
 
     def open(
         self, path_info, mode="r", **kwargs
@@ -56,6 +96,9 @@ class FSSpecWrapper(BaseFileSystem):
     def ls(
         self, path_info, detail=False, recursive=False
     ):  # pylint: disable=arguments-differ
+        if self.isdir(path_info) and self.is_empty(path_info):
+            return None
+
         path = self._with_bucket(path_info)
         if recursive:
             for root, _, files in self.fs.walk(path, detail=detail):

dvc/fs/gdrive.py

@@ -126,6 +126,9 @@ class GDriveFileSystem(BaseFileSystem):
         self._client_id = config.get("gdrive_client_id")
         self._client_secret = config.get("gdrive_client_secret")
         self._validate_config()
+        self._gdrive_service_credentials_path = tmp_fname(
+            os.path.join(self.repo.tmp_dir, "")
+        )
         self._gdrive_user_credentials_path = (
             tmp_fname(os.path.join(self.repo.tmp_dir, ""))
             if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
@@ -208,8 +211,15 @@ class GDriveFileSystem(BaseFileSystem):
         from pydrive2.auth import GoogleAuth
         from pydrive2.drive import GoogleDrive
 
-        if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
-            with open(self._gdrive_user_credentials_path, "w") as cred_file:
+        temporary_save_path = self._gdrive_user_credentials_path
+        is_credentials_temp = os.getenv(
+            GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
+        )
+        if self._use_service_account:
+            temporary_save_path = self._gdrive_service_credentials_path
+
+        if is_credentials_temp:
+            with open(temporary_save_path, "w") as cred_file:
                 cred_file.write(
                     os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
                 )
@@ -217,6 +227,7 @@ class GDriveFileSystem(BaseFileSystem):
         auth_settings = {
             "client_config_backend": "settings",
             "save_credentials": True,
+            "save_credentials_backend": "file",
             "save_credentials_file": self._gdrive_user_credentials_path,
             "get_refresh_token": True,
             "oauth_scope": [
@@ -226,13 +237,16 @@ class GDriveFileSystem(BaseFileSystem):
         }
 
         if self._use_service_account:
-            auth_settings["save_credentials_backend"] = "json"
             auth_settings["service_config"] = {
                 "client_user_email": self._service_account_user_email,
                 "client_json_file_path": self._service_account_json_file_path,
             }
+            if is_credentials_temp:
+                auth_settings["service_config"][
+                    "client_json_file_path"
+                ] = temporary_save_path
+
         else:
-            auth_settings["save_credentials_backend"] = "file"
             auth_settings["client_config"] = {
                 "client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
                 "client_secret": self._client_secret
@@ -267,8 +281,8 @@ class GDriveFileSystem(BaseFileSystem):
         except Exception as exc:
             raise GDriveAuthError(self.credentials_location) from exc
         finally:
-            if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
-                os.remove(self._gdrive_user_credentials_path)
+            if is_credentials_temp:
+                os.remove(temporary_save_path)
 
         return GoogleDrive(gauth)
 

dvc/fs/git.py

@@ -158,12 +158,6 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 errno.ENOENT, os.strerror(errno.ENOENT), path
             )
 
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        # NOTE: gitpython is not threadsafe. See
-        # https://github.com/iterative/dvc/issues/4079
-        return 1
-
     def walk_files(self, top):  # pylint: disable=arguments-differ
         for root, _, files in self.walk(top):
             for file in files:

dvc/fs/gs.py

@@ -22,16 +22,15 @@ class GSFileSystem(FSSpecWrapper):
         url = config.get("url", "gs://")
         self.path_info = self.PATH_CLS(url)
 
-        self.login_info = self._prepare_credentials(config)
-
     def _prepare_credentials(self, config):
-        login_info = {}
+        login_info = {"consistency": None}
         login_info["project"] = config.get("projectname")
         login_info["token"] = config.get("credentialpath")
         return login_info
 
     def _entry_hook(self, entry):
-        entry["etag"] = base64.b64decode(entry["etag"]).hex()
+        if "etag" in entry:
+            entry["etag"] = base64.b64decode(entry["etag"]).hex()
         return entry
 
     @wrap_prop(threading.Lock())
@@ -39,4 +38,4 @@ class GSFileSystem(FSSpecWrapper):
     def fs(self):
         from gcsfs import GCSFileSystem
 
-        return GCSFileSystem(**self.login_info)
+        return GCSFileSystem(**self.fs_args)

dvc/fs/http.py

@@ -1,6 +1,7 @@
 import logging
 import os.path
 import threading
+from typing import Optional
 
 from funcy import cached_property, memoize, wrap_prop, wrap_with
 
@@ -146,10 +147,9 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         raise HTTPError(res.status_code, res.reason)
 
     def info(self, path_info):
-        url = path_info.url
-        headers = self._head(url).headers
-        etag = headers.get("ETag") or headers.get("Content-MD5")
-        size = int(headers.get("Content-Length"))
+        resp = self._head(path_info.url)
+        etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5")
+        size = self._content_length(resp)
         return {"etag": etag, "size": size}
 
     def _upload_fobj(self, fobj, to_info):
@@ -195,6 +195,6 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
     @staticmethod
-    def _content_length(response):
+    def _content_length(response) -> Optional[int]:
         res = response.headers.get("Content-Length")
         return int(res) if res else None

dvc/fs/memory.py

@@ -13,13 +13,16 @@ class MemoryFileSystem(BaseFileSystem):
         self.fs = MemFS()
 
     def exists(self, path_info, use_dvcignore=True):
-        return self.fs.exists(path_info.path)
+        return self.fs.exists(path_info.fspath)
 
     def open(self, path_info, mode="r", encoding=None, **kwargs):
         return self.fs.open(
             path_info.fspath, mode=mode, encoding=encoding, **kwargs
         )
 
+    def info(self, path_info):
+        return self.fs.info(path_info.fspath)
+
     def stat(self, path_info):
         import os
 

dvc/fs/s3.py

@@ -261,8 +261,13 @@ class S3FileSystem(BaseFileSystem):
         return path_info.path in self._list_paths(path_info)
 
     def info(self, path_info):
+        # temporary workaround, will be removed when migrating to s3fs
+        if self.isdir(path_info):
+            return {"type": "directory"}
+
         with self._get_obj(path_info) as obj:
             return {
+                "type": "file",
                 "etag": obj.e_tag.strip('"'),
                 "size": obj.content_length,
             }

dvc/fs/ssh/connection.py

@@ -40,7 +40,9 @@ class SSHConnection:
                 host=host, **kwargs
             )
         )
-        self.timeout = kwargs.get("timeout", 1800)
+
+        kwargs.setdefault("timeout", 1800)
+        self.timeout = kwargs["timeout"]
 
         self._ssh = paramiko.SSHClient()
 

dvc/hash_info.py

@@ -1,11 +1,9 @@
 from collections import OrderedDict
 from dataclasses import dataclass, field
-from typing import Dict, Optional
+from typing import Optional
 
 HASH_DIR_SUFFIX = ".dir"
 
-DirInfo = Dict[str, str]
-
 
 @dataclass
 class HashInfo:
@@ -14,7 +12,6 @@ class HashInfo:
 
     name: Optional[str]
     value: Optional[str]
-    dir_info: Optional[DirInfo] = field(default=None, compare=False)
     size: Optional[int] = field(default=None, compare=False)
     nfiles: Optional[int] = field(default=None, compare=False)
 

dvc/logger.py

@@ -23,14 +23,19 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     Adds a new logging level to the `logging` module and the
     currently configured logging class.
 
+    Uses the existing numeric levelNum if already defined.
+
     Based on https://stackoverflow.com/questions/2183233
     """
     if methodName is None:
         methodName = levelName.lower()
 
-    assert not hasattr(logging, levelName)
-    assert not hasattr(logging, methodName)
-    assert not hasattr(logging.getLoggerClass(), methodName)
+    # If the level name is already defined as a top-level `logging`
+    # constant, then adopt the existing numeric level.
+    if hasattr(logging, levelName):
+        existingLevelNum = getattr(logging, levelName)
+        assert isinstance(existingLevelNum, int)
+        levelNum = existingLevelNum
 
     def logForLevel(self, message, *args, **kwargs):
         if self.isEnabledFor(levelNum):
@@ -40,10 +45,19 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     def logToRoot(message, *args, **kwargs):
         logging.log(levelNum, message, *args, **kwargs)
 
-    logging.addLevelName(levelNum, levelName)
-    setattr(logging, levelName, levelNum)
-    setattr(logging.getLoggerClass(), methodName, logForLevel)
-    setattr(logging, methodName, logToRoot)
+    # getLevelName resolves the numeric log level if already defined,
+    # otherwise returns a string
+    if not isinstance(logging.getLevelName(levelName), int):
+        logging.addLevelName(levelNum, levelName)
+
+    if not hasattr(logging, levelName):
+        setattr(logging, levelName, levelNum)
+
+    if not hasattr(logging.getLoggerClass(), methodName):
+        setattr(logging.getLoggerClass(), methodName, logForLevel)
+
+    if not hasattr(logging, methodName):
+        setattr(logging, methodName, logToRoot)
 
 
 class LoggingException(Exception):

dvc/main.py

@@ -45,6 +45,11 @@ def main(argv=None):  # noqa: C901
 
         logger.trace(args)
 
+        if not args.quiet:
+            from dvc.ui import ui
+
+            ui.enable()
+
         with debugtools(args):
             cmd = args.func(args)
             ret = cmd.run()

dvc/objects/__init__.py

@@ -1,315 +1,28 @@
-import itertools
-import json
 import logging
-from concurrent import futures
 
-from dvc.dir_info import DirInfo
-from dvc.exceptions import DvcException
 from dvc.progress import Tqdm
 
-from .stage import get_hash
+from .tree import Tree
 
 logger = logging.getLogger(__name__)
 
 
-class ObjectError(DvcException):
-    pass
-
-
-class ObjectFormatError(ObjectError):
-    pass
-
-
-class HashFile:
-    def __init__(self, path_info, fs, hash_info):
-        self.path_info = path_info
-        self.fs = fs
-        self.hash_info = hash_info
-
-    def __len__(self):
-        return 1
-
-    def __str__(self):
-        return f"object {self.hash_info}"
-
-    def __bool__(self):
-        return bool(self.hash_info)
-
-    def check(self, odb):
-        actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM, odb)
-
-        logger.trace(
-            "cache '%s' expected '%s' actual '%s'",
-            self.path_info,
-            self.hash_info,
-            actual,
-        )
-
-        assert actual.name == self.hash_info.name
-        if actual.value.split(".")[0] != self.hash_info.value.split(".")[0]:
-            raise ObjectFormatError(f"{self} is corrupted")
-
-    @classmethod
-    def load(cls, odb, hash_info):
-        return odb.get(hash_info)
-
-    def save(self, odb, **kwargs):
-        odb.add(self.path_info, self.fs, self.hash_info, **kwargs)
-
-
-class File(HashFile):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.src = None
-
-    @classmethod
-    def stage(cls, odb, path_info, fs, **kwargs):
-        hash_info = get_hash(
-            path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
-        )
-        raw = odb.get(hash_info)
-        obj = cls(raw.path_info, raw.fs, hash_info)
-        obj.src = HashFile(path_info, fs, hash_info)
-        return obj
-
-    def save(self, odb, **kwargs):
-        self.src.save(odb, **kwargs)
-
-
-class Tree(HashFile):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.src_hash_info = None
-        self.src_path_info = None
-        self.src_fs = None
-
-    def __len__(self):
-        return self.hash_info.dir_info.nfiles
-
-    def __iter__(self):
-        yield from self.hash_info.dir_info.items()
-
-    @classmethod
-    def save_dir_info(cls, odb, dir_info, hash_info=None):
-        if hash_info and hash_info.name == odb.fs.PARAM_CHECKSUM:
-            try:
-                odb.check(hash_info)
-                assert hash_info.dir_info == dir_info
-                return hash_info
-            except (FileNotFoundError, ObjectFormatError):
-                pass
-
-        from dvc.fs.memory import MemoryFileSystem
-        from dvc.path_info import PathInfo
-        from dvc.utils import tmp_fname
-
-        fs = MemoryFileSystem(None, {})
-        path_info = PathInfo(tmp_fname(""))
-        with fs.open(path_info, "w") as fobj:
-            json.dump(dir_info.to_list(), fobj, sort_keys=True)
-
-        tmp_info = odb.fs.path_info / tmp_fname("")
-        with fs.open(path_info, "rb") as fobj:
-            odb.fs.upload_fobj(fobj, tmp_info)
-
-        hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM, odb)
-        hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX
-        hash_info.dir_info = dir_info
-        hash_info.nfiles = dir_info.nfiles
-
-        odb.add(tmp_info, odb.fs, hash_info)
-
-        return hash_info
-
-    @classmethod
-    def stage(cls, odb, path_info, fs, **kwargs):
-        hash_info = get_hash(
-            path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
-        )
-        hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info)
-        hi.size = hash_info.size
-        raw = odb.get(hi)
-        obj = cls(raw.path_info, raw.fs, hi)
-        obj.src_hash_info = hash_info
-        obj.src_path_info = path_info
-        obj.src_fs = fs
-        return obj
-
-    @classmethod
-    def load(cls, odb, hash_info):
-
-        obj = odb.get(hash_info)
-
-        try:
-            with obj.fs.open(obj.path_info, "r") as fobj:
-                raw = json.load(fobj)
-        except ValueError as exc:
-            raise ObjectFormatError(f"{obj} is corrupted") from exc
-
-        if not isinstance(raw, list):
-            logger.error(
-                "dir cache file format error '%s' [skipping the file]",
-                obj.path_info,
-            )
-            raise ObjectFormatError(f"{obj} is corrupted")
-
-        dir_info = DirInfo.from_list(raw)
-        hash_info.dir_info = dir_info
-        hash_info.nfiles = dir_info.nfiles
-
-        return cls(obj.path_info, obj.fs, hash_info)
-
-    def save(self, odb, **kwargs):
-        assert self.src_hash_info.dir_info
-        hi = self.save_dir_info(
-            odb, self.src_hash_info.dir_info, self.hash_info
-        )
-        for entry_key, entry_hash in Tqdm(
-            hi.dir_info.items(),
-            desc="Saving " + self.src_path_info.name,
-            unit="file",
-        ):
-            entry_info = self.src_path_info.joinpath(*entry_key)
-            entry_obj = HashFile(entry_info, self.src_fs, entry_hash)
-            entry_obj.save(odb, **kwargs)
-        self.src_fs.repo.state.save(self.src_path_info, self.src_fs, hi)
-
-    def filter(self, odb, prefix):
-        hash_info = self.hash_info.dir_info.get(prefix)
-        if hash_info:
-            return load(odb, hash_info)
-
-        depth = len(prefix)
-        dir_info = DirInfo()
-        try:
-            for key, value in self.hash_info.dir_info.trie.items(prefix):
-                dir_info.add(key[depth:], value)
-        except KeyError:
-            return None
-
-        return load(odb, self.save_dir_info(odb, dir_info))
-
-
 def save(odb, obj, **kwargs):
-    obj.save(odb, **kwargs)
+    if isinstance(obj, Tree):
+        for _, entry in Tqdm(obj):
+            odb.add(entry.path_info, entry.fs, entry.hash_info, **kwargs)
+    odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)
 
 
 def check(odb, obj):
     odb.check(obj.hash_info)
 
     if isinstance(obj, Tree):
-        for _, hash_info in obj:
-            odb.check(hash_info)
+        for _, entry in obj:
+            odb.check(entry.hash_info)
 
 
 def load(odb, hash_info):
     if hash_info.isdir:
         return Tree.load(odb, hash_info)
-    return File.load(odb, hash_info)
-
-
-def _get_dir_size(odb, dir_info):
-    try:
-        return sum(
-            odb.fs.getsize(odb.hash_to_path_info(hi.value))
-            for _, hi in dir_info.items()
-        )
-    except FileNotFoundError:
-        return None
-
-
-def merge(odb, ancestor_info, our_info, their_info):
-    assert our_info
-    assert their_info
-
-    if ancestor_info:
-        ancestor = load(odb, ancestor_info).hash_info.dir_info
-    else:
-        ancestor = DirInfo()
-
-    our = load(odb, our_info).hash_info.dir_info
-    their = load(odb, their_info).hash_info.dir_info
-
-    merged = our.merge(ancestor, their)
-    hash_info = Tree.save_dir_info(odb, merged)
-    hash_info.size = _get_dir_size(odb, merged)
-    return hash_info
-
-
-def _transfer_file(odb, from_fs, from_info):
-    from dvc.utils import tmp_fname
-    from dvc.utils.stream import HashedStreamReader
-
-    tmp_info = odb.fs.path_info / tmp_fname()
-    with from_fs.open(
-        from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE
-    ) as stream:
-        stream_reader = HashedStreamReader(stream)
-        # Since we don't know the hash beforehand, we'll
-        # upload it to a temporary location and then move
-        # it.
-        odb.fs.upload_fobj(
-            stream_reader,
-            tmp_info,
-            total=from_fs.getsize(from_info),
-            desc=from_info.name,
-        )
-
-    hash_info = stream_reader.hash_info
-    return tmp_info, hash_info
-
-
-def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-    rel_path_infos = {}
-    from_infos = from_fs.walk_files(from_info)
-
-    def create_tasks(executor, amount):
-        for entry_info in itertools.islice(from_infos, amount):
-            pbar.total += 1
-            task = executor.submit(
-                pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info
-            )
-            rel_path_infos[task] = entry_info.relative_to(from_info)
-            yield task
-
-    pbar.total = 0
-    with futures.ThreadPoolExecutor(max_workers=jobs) as executor:
-        tasks = set(create_tasks(executor, jobs * 5))
-
-        while tasks:
-            done, tasks = futures.wait(
-                tasks, return_when=futures.FIRST_COMPLETED
-            )
-            tasks.update(create_tasks(executor, len(done)))
-            for task in done:
-                yield rel_path_infos.pop(task), task.result()
-
-
-def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
-    dir_info = DirInfo()
-
-    with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
-        for (
-            entry_info,
-            (entry_tmp_info, entry_hash),
-        ) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-            odb.add(entry_tmp_info, odb.fs, entry_hash)
-            dir_info.add(entry_info.parts, entry_hash)
-
-    return Tree.save_dir_info(odb, dir_info)
-
-
-def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
-    jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
-
-    if from_fs.isdir(from_info):
-        return _transfer_directory(
-            odb,
-            from_fs,
-            from_info,
-            jobs=jobs,
-            no_progress_bar=no_progress_bar,
-        )
-    tmp_info, hash_info = _transfer_file(odb, from_fs, from_info)
-    odb.add(tmp_info, odb.fs, hash_info)
-    return hash_info
+    return odb.get(hash_info)

dvc/objects/db/base.py

@@ -4,10 +4,10 @@ from concurrent.futures import ThreadPoolExecutor
 from copy import copy
 from typing import Optional
 
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.file import HashFile
 from dvc.progress import Tqdm
 
-from .. import HashFile, ObjectFormatError
-
 logger = logging.getLogger(__name__)
 
 

dvc/objects/db/local.py

@@ -6,7 +6,7 @@ from funcy import cached_property
 from shortuuid import uuid
 
 from dvc.hash_info import HashInfo
-from dvc.objects import ObjectFormatError
+from dvc.objects.errors import ObjectFormatError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
 from dvc.utils import relpath

dvc/objects/errors.py

@@ -0,0 +1,9 @@
+from dvc.exceptions import DvcException
+
+
+class ObjectError(DvcException):
+    pass
+
+
+class ObjectFormatError(ObjectError):
+    pass

dvc/objects/file.py

@@ -0,0 +1,53 @@
+import logging
+
+from .errors import ObjectFormatError
+from .stage import get_file_hash
+
+logger = logging.getLogger(__name__)
+
+
+class HashFile:
+    def __init__(self, path_info, fs, hash_info):
+        self.path_info = path_info
+        self.fs = fs
+        self.hash_info = hash_info
+
+    @property
+    def size(self):
+        if not (self.path_info and self.fs):
+            return None
+        return self.fs.getsize(self.path_info)
+
+    def __len__(self):
+        return 1
+
+    def __str__(self):
+        return f"object {self.hash_info}"
+
+    def __bool__(self):
+        return bool(self.hash_info)
+
+    def __eq__(self, other):
+        if not isinstance(other, HashFile):
+            return False
+        return (
+            self.path_info == other.path_info
+            and self.fs == other.fs
+            and self.hash_info == other.hash_info
+        )
+
+    def check(self, odb):
+        actual = get_file_hash(
+            self.path_info, self.fs, self.hash_info.name, odb.repo.state
+        )
+
+        logger.trace(
+            "cache '%s' expected '%s' actual '%s'",
+            self.path_info,
+            self.hash_info,
+            actual,
+        )
+
+        assert actual.name == self.hash_info.name
+        if actual.value.split(".")[0] != self.hash_info.value.split(".")[0]:
+            raise ObjectFormatError(f"{self} is corrupted")

dvc/objects/stage.py

@@ -9,7 +9,7 @@ from dvc.progress import Tqdm
 from dvc.utils import file_md5
 
 
-def get_file_hash(path_info, fs, name):
+def _get_file_hash(path_info, fs, name):
     info = fs.info(path_info)
     if name in info:
         assert not info[name].endswith(".dir")
@@ -27,20 +27,36 @@ def get_file_hash(path_info, fs, name):
     raise NotImplementedError
 
 
-def _calculate_hashes(path_info, fs, name, state, **kwargs):
-    def _get_file_hash(file_info):
+def get_file_hash(path_info, fs, name, state=None):
+    if state:
         hash_info = state.get(  # pylint: disable=assignment-from-none
-            file_info, fs,
+            path_info, fs,
         )
-        if not hash_info:
-            hash_info = get_file_hash(file_info, fs, name)
-            state.save(file_info, fs, hash_info)
-        return file_info, hash_info
+        if hash_info:
+            return hash_info
+
+    if not fs.exists(path_info):
+        raise FileNotFoundError(
+            errno.ENOENT, os.strerror(errno.ENOENT), path_info
+        )
+
+    hash_info = _get_file_hash(path_info, fs, name)
+
+    if state:
+        assert ".dir" not in hash_info.value
+        state.save(path_info, fs, hash_info)
+
+    return hash_info
+
+
+def _calculate_hashes(path_info, fs, name, state, **kwargs):
+    def _hash(file_info):
+        return file_info, get_file_hash(file_info, fs, name, state)
 
     with Tqdm(
         unit="md5", desc="Computing file/dir hashes (only done once)",
     ) as pbar:
-        worker = pbar.wrap_fn(_get_file_hash)
+        worker = pbar.wrap_fn(_hash)
         with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
             pairs = executor.map(worker, fs.walk_files(path_info, **kwargs))
             return dict(pairs)
@@ -60,14 +76,17 @@ def _iter_hashes(path_info, fs, name, state, **kwargs):
     yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items()
 
 
-def _collect_dir(path_info, fs, name, state, **kwargs):
-    from dvc.dir_info import DirInfo
+def _build_tree(path_info, fs, name, state, **kwargs):
+    from .file import HashFile
+    from .tree import Tree
 
-    dir_info = DirInfo()
+    tree = Tree(None, None, None)
     for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs):
         if DvcIgnore.DVCIGNORE_FILE == fi.name:
             raise DvcIgnoreInCollectedDirError(fi.parent)
 
+        obj = HashFile(fi, fs, hi)
+
         # NOTE: this is lossy transformation:
         #   "hey\there" -> "hey/there"
         #   "hey/there" -> "hey/there"
@@ -76,22 +95,44 @@ def _collect_dir(path_info, fs, name, state, **kwargs):
         #
         # Yes, this is a BUG, as long as we permit "/" in
         # filenames on Windows and "\" on Unix
-        dir_info.add(fi.relative_to(path_info).parts, hi)
+        tree.add(fi.relative_to(path_info).parts, obj)
 
-    return dir_info
+    tree.digest()
+
+    return tree
 
 
-def get_dir_hash(path_info, fs, name, state, **kwargs):
-    from . import Tree
+def _get_tree_obj(path_info, fs, name, odb, state, **kwargs):
+    from .tree import Tree
 
-    dir_info = _collect_dir(path_info, fs, name, state, **kwargs)
-    hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info)
-    hash_info.size = dir_info.size
-    hash_info.dir_info = dir_info
-    return hash_info
+    value = fs.info(path_info).get(name)
+    if value:
+        hash_info = HashInfo(name, value)
+        try:
+            return Tree.load(odb, hash_info)
+        except FileNotFoundError:
+            pass
+
+    tree = _build_tree(path_info, fs, name, state, **kwargs)
+
+    odb.add(tree.path_info, tree.fs, tree.hash_info)
+    if name != "md5":
+        # NOTE: used only for external outputs. Initial reasoning was to be
+        # able to validate .dir files right in the workspace (e.g. check s3
+        # etag), but could be dropped for manual validation with regular md5,
+        # that would be universal for all clouds.
+        raw = odb.get(tree.hash_info)
+        hash_info = get_file_hash(raw.path_info, raw.fs, name, state)
+        tree.hash_info.name = hash_info.name
+        tree.hash_info.value = hash_info.value
+        if not tree.hash_info.value.endswith(".dir"):
+            tree.hash_info.value += ".dir"
+        odb.add(tree.path_info, tree.fs, tree.hash_info)
+
+    return tree
 
 
-def get_hash(path_info, fs, name, odb, **kwargs):
+def stage(odb, path_info, fs, name, **kwargs):
     assert path_info and (
         isinstance(path_info, str) or path_info.scheme == fs.scheme
     )
@@ -116,28 +157,31 @@ def get_hash(path_info, fs, name, odb, **kwargs):
         hash_info = None
 
     if hash_info:
-        if hash_info.isdir:
-            from . import Tree
+        from . import load
+        from .tree import Tree
 
-            # NOTE: loading the fs will restore hash_info.dir_info
-            Tree.load(odb, hash_info)
-        assert hash_info.name == name
-        return hash_info
+        obj = load(odb, hash_info)
+        if isinstance(obj, Tree):
+            obj.hash_info.nfiles = len(obj)
+            for key, entry in obj:
+                entry.fs = fs
+                entry.path_info = path_info.joinpath(*key)
+        else:
+            obj.fs = fs
+            obj.path_info = path_info
+        assert obj.hash_info.name == name
+        obj.hash_info.size = hash_info.size
+        return obj
 
     if fs.isdir(path_info):
-        hash_info = get_dir_hash(path_info, fs, name, state, **kwargs)
+        obj = _get_tree_obj(path_info, fs, name, odb, state, **kwargs)
     else:
-        hash_info = get_file_hash(path_info, fs, name)
+        from .file import HashFile
 
-    if hash_info and fs.exists(path_info):
-        state.save(path_info, fs, hash_info)
+        hash_info = get_file_hash(path_info, fs, name, state)
+        obj = HashFile(path_info, fs, hash_info)
 
-    return hash_info
+    if obj.hash_info and fs.exists(path_info):
+        state.save(path_info, fs, obj.hash_info)
 
-
-def stage(odb, path_info, fs, **kwargs):
-    from . import File, Tree
-
-    if fs.isdir(path_info):
-        return Tree.stage(odb, path_info, fs, **kwargs)
-    return File.stage(odb, path_info, fs, **kwargs)
+    return obj

dvc/objects/transfer.py

@@ -0,0 +1,92 @@
+import itertools
+import logging
+from concurrent import futures
+
+from dvc.progress import Tqdm
+
+from . import save
+from .file import HashFile
+from .tree import Tree
+
+logger = logging.getLogger(__name__)
+
+
+def _transfer_file(odb, from_fs, from_info):
+    from dvc.utils import tmp_fname
+    from dvc.utils.stream import HashedStreamReader
+
+    tmp_info = odb.fs.path_info / tmp_fname()
+    with from_fs.open(
+        from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE
+    ) as stream:
+        stream_reader = HashedStreamReader(stream)
+        # Since we don't know the hash beforehand, we'll
+        # upload it to a temporary location and then move
+        # it.
+        odb.fs.upload_fobj(
+            stream_reader,
+            tmp_info,
+            total=from_fs.getsize(from_info),
+            desc=from_info.name,
+        )
+
+    hash_info = stream_reader.hash_info
+    return tmp_info, hash_info
+
+
+def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
+    rel_path_infos = {}
+    from_infos = from_fs.walk_files(from_info)
+
+    def create_tasks(executor, amount):
+        for entry_info in itertools.islice(from_infos, amount):
+            pbar.total += 1
+            task = executor.submit(
+                pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info
+            )
+            rel_path_infos[task] = entry_info.relative_to(from_info)
+            yield task
+
+    pbar.total = 0
+    with futures.ThreadPoolExecutor(max_workers=jobs) as executor:
+        tasks = set(create_tasks(executor, jobs * 5))
+
+        while tasks:
+            done, tasks = futures.wait(
+                tasks, return_when=futures.FIRST_COMPLETED
+            )
+            tasks.update(create_tasks(executor, len(done)))
+            for task in done:
+                yield rel_path_infos.pop(task), task.result()
+
+
+def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
+    tree = Tree(None, None, None)
+
+    with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
+        for (
+            entry_info,
+            (entry_tmp_info, entry_hash),
+        ) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
+            obj = HashFile(entry_tmp_info, odb.fs, entry_hash)
+            tree.add(entry_info.parts, obj)
+
+    tree.digest()
+    save(odb, tree)
+    return tree.hash_info
+
+
+def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
+    jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
+
+    if from_fs.isdir(from_info):
+        return _transfer_directory(
+            odb,
+            from_fs,
+            from_info,
+            jobs=jobs,
+            no_progress_bar=no_progress_bar,
+        )
+    tmp_info, hash_info = _transfer_file(odb, from_fs, from_info)
+    odb.add(tmp_info, odb.fs, hash_info)
+    return hash_info

dvc/objects/tree.py

@@ -0,0 +1,211 @@
+import json
+import logging
+import posixpath
+
+from funcy import cached_property
+
+from .errors import ObjectFormatError
+from .file import HashFile
+from .stage import get_file_hash
+
+logger = logging.getLogger(__name__)
+
+
+class Tree(HashFile):
+    PARAM_RELPATH = "relpath"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._dict = {}
+
+    @cached_property
+    def trie(self):
+        from pygtrie import Trie
+
+        return Trie(self._dict)
+
+    @property
+    def size(self):
+        try:
+            return sum(obj.size for _, obj in self)
+        except TypeError:
+            return None
+
+    def add(self, key, obj):
+        self.__dict__.pop("trie", None)
+        self._dict[key] = obj
+
+    def digest(self):
+        from dvc.fs.memory import MemoryFileSystem
+        from dvc.path_info import PathInfo
+        from dvc.utils import tmp_fname
+
+        memfs = MemoryFileSystem(None, {})
+        path_info = PathInfo(tmp_fname(""))
+        with memfs.open(path_info, "wb") as fobj:
+            fobj.write(self.as_bytes())
+        self.fs = memfs
+        self.path_info = path_info
+        self.hash_info = get_file_hash(path_info, memfs, "md5")
+        self.hash_info.value += ".dir"
+        self.hash_info.size = self.size
+        self.hash_info.nfiles = len(self)
+
+    def __len__(self):
+        return len(self._dict)
+
+    def __iter__(self):
+        yield from self._dict.items()
+
+    def as_dict(self):
+        return self._dict.copy()
+
+    def as_list(self):
+        from operator import itemgetter
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(
+            (
+                {
+                    # NOTE: not using hash_info.to_dict() because we don't want
+                    # size/nfiles fields at this point.
+                    obj.hash_info.name: obj.hash_info.value,
+                    self.PARAM_RELPATH: posixpath.sep.join(parts),
+                }
+                for parts, obj in self._dict.items()  # noqa: B301
+            ),
+            key=itemgetter(self.PARAM_RELPATH),
+        )
+
+    def as_bytes(self):
+        return json.dumps(self.as_list(), sort_keys=True).encode("utf-8")
+
+    @classmethod
+    def from_list(cls, lst):
+        from dvc.hash_info import HashInfo
+
+        tree = cls(None, None, None)
+        for _entry in lst:
+            entry = _entry.copy()
+            relpath = entry.pop(cls.PARAM_RELPATH)
+            parts = tuple(relpath.split(posixpath.sep))
+            hash_info = HashInfo.from_dict(entry)
+            obj = HashFile(None, None, hash_info)
+            tree.add(parts, obj)
+        return tree
+
+    @classmethod
+    def load(cls, odb, hash_info):
+
+        obj = odb.get(hash_info)
+
+        try:
+            with obj.fs.open(obj.path_info, "r") as fobj:
+                raw = json.load(fobj)
+        except ValueError as exc:
+            raise ObjectFormatError(f"{obj} is corrupted") from exc
+
+        if not isinstance(raw, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                obj.path_info,
+            )
+            raise ObjectFormatError(f"{obj} is corrupted")
+
+        tree = cls.from_list(raw)
+        tree.path_info = obj.path_info
+        tree.fs = obj.fs
+        tree.hash_info = hash_info
+
+        return tree
+
+    def filter(self, odb, prefix):
+        obj = self._dict.get(prefix)
+        if obj:
+            return obj
+
+        depth = len(prefix)
+        tree = Tree(None, None, None)
+        try:
+            for key, obj in self.trie.items(prefix):
+                tree.add(key[depth:], obj)
+        except KeyError:
+            return None
+        tree.digest()
+        odb.add(tree.path_info, tree.fs, tree.hash_info)
+        return tree
+
+
+def _get_dir_size(odb, tree):
+    try:
+        return sum(
+            odb.fs.getsize(odb.hash_to_path_info(obj.hash_info.value))
+            for _, obj in tree
+        )
+    except FileNotFoundError:
+        return None
+
+
+def _diff(ancestor, other, allow_removed=False):
+    from dictdiffer import diff
+
+    from dvc.exceptions import MergeError
+
+    allowed = ["add"]
+    if allow_removed:
+        allowed.append("remove")
+
+    result = list(diff(ancestor, other))
+    for typ, _, _ in result:
+        if typ not in allowed:
+            raise MergeError(
+                "unable to auto-merge directories with diff that contains "
+                f"'{typ}'ed files"
+            )
+    return result
+
+
+def _merge(ancestor, our, their):
+    import copy
+
+    from dictdiffer import patch
+
+    our_diff = _diff(ancestor, our)
+    if not our_diff:
+        return copy.deepcopy(their)
+
+    their_diff = _diff(ancestor, their)
+    if not their_diff:
+        return copy.deepcopy(our)
+
+    # make sure there are no conflicting files
+    _diff(our, their, allow_removed=True)
+
+    return patch(our_diff + their_diff, ancestor)
+
+
+def merge(odb, ancestor_info, our_info, their_info):
+    from . import load
+
+    assert our_info
+    assert their_info
+
+    if ancestor_info:
+        ancestor = load(odb, ancestor_info)
+    else:
+        ancestor = Tree(None, None, None)
+
+    our = load(odb, our_info)
+    their = load(odb, their_info)
+
+    merged_dict = _merge(ancestor.as_dict(), our.as_dict(), their.as_dict(),)
+
+    merged = Tree(None, None, None)
+    for key, hi in merged_dict.items():
+        merged.add(key, hi)
+    merged.digest()
+
+    odb.add(merged.path_info, merged.fs, merged.hash_info)
+    hash_info = merged.hash_info
+    hash_info.size = _get_dir_size(odb, merged)
+    return hash_info

dvc/output/base.py

@@ -18,7 +18,7 @@ from dvc.exceptions import (
 )
 from dvc.hash_info import HashInfo
 from dvc.objects.db import NamedCache
-from dvc.objects.stage import get_hash
+from dvc.objects.errors import ObjectFormatError
 from dvc.objects.stage import stage as ostage
 
 from ..fs.base import BaseFileSystem
@@ -183,23 +183,21 @@ class BaseOutput:
     def odb(self):
         return getattr(self.repo.odb, self.scheme)
 
-    @property
-    def dir_cache(self):
-        return self.hash_info.dir_info
-
     @property
     def cache_path(self):
         return self.odb.hash_to_path_info(self.hash_info.value).url
 
     def get_hash(self):
         if not self.use_cache:
-            return get_hash(
+            return ostage(
+                self.repo.odb.local,
                 self.path_info,
                 self.fs,
                 self.fs.PARAM_CHECKSUM,
-                self.repo.odb.local,
-            )
-        return ostage(self.odb, self.path_info, self.fs).hash_info
+            ).hash_info
+        return ostage(
+            self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
+        ).hash_info
 
     @property
     def is_dir_checksum(self):
@@ -223,7 +221,7 @@ class BaseOutput:
         try:
             objects.check(self.odb, obj)
             return False
-        except (FileNotFoundError, objects.ObjectFormatError):
+        except (FileNotFoundError, ObjectFormatError):
             return True
 
     def workspace_status(self):
@@ -307,7 +305,9 @@ class BaseOutput:
             logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
-        self.obj = ostage(self.odb, self.path_info, self.fs)
+        self.obj = ostage(
+            self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
+        )
         self.hash_info = self.obj.hash_info
         self.isexec = self.isfile() and self.fs.isexec(self.path_info)
 
@@ -322,7 +322,12 @@ class BaseOutput:
         assert self.hash_info
 
         if self.use_cache:
-            obj = ostage(self.odb, filter_info or self.path_info, self.fs)
+            obj = ostage(
+                self.odb,
+                filter_info or self.path_info,
+                self.fs,
+                self.odb.fs.PARAM_CHECKSUM,
+            )
             objects.save(self.odb, obj)
             checkout(
                 filter_info or self.path_info,
@@ -403,6 +408,7 @@ class BaseOutput:
         relink=False,
         filter_info=None,
         allow_missing=False,
+        checkpoint_reset=False,
         **kwargs,
     ):
         if not self.use_cache:
@@ -417,6 +423,11 @@ class BaseOutput:
             # backward compatibility
             return None
 
+        if self.checkpoint and checkpoint_reset:
+            if self.exists:
+                self.remove()
+            return None
+
         added = not self.exists
 
         try:
@@ -460,8 +471,6 @@ class BaseOutput:
             self.repo.scm.ignore(self.fspath)
 
     def get_files_number(self, filter_info=None):
-        from funcy import ilen
-
         if not self.use_cache or not self.hash_info:
             return 0
 
@@ -471,13 +480,8 @@ class BaseOutput:
         if not filter_info or filter_info == self.path_info:
             return self.hash_info.nfiles or 0
 
-        if not self.hash_info.dir_info:
-            return 0
-
-        return ilen(
-            filter_info.isin_or_eq(self.path_info / relpath)
-            for relpath, _ in self.hash_info.dir_info.items()
-        )
+        obj = self.get_obj(filter_info=filter_info)
+        return len(obj) if obj else 0
 
     def unprotect(self):
         if self.exists:
@@ -490,7 +494,7 @@ class BaseOutput:
 
         try:
             objects.check(self.odb, self.odb.get(self.hash_info))
-        except (FileNotFoundError, objects.ObjectFormatError):
+        except (FileNotFoundError, ObjectFormatError):
             self.repo.cloud.pull(
                 NamedCache.make("local", self.hash_info.value, str(self)),
                 show_checksums=False,
@@ -498,12 +502,11 @@ class BaseOutput:
             )
 
         try:
-            objects.load(self.odb, self.hash_info)
-            assert self.hash_info.dir_info
-        except (objects.ObjectFormatError, FileNotFoundError):
-            self.hash_info.dir_info = None
+            self.obj = objects.load(self.odb, self.hash_info)
+        except (FileNotFoundError, ObjectFormatError):
+            self.obj = None
 
-        return self.dir_cache
+        return self.obj
 
     def collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
@@ -536,7 +539,7 @@ class BaseOutput:
 
         try:
             objects.check(self.odb, self.odb.get(self.hash_info))
-        except (FileNotFoundError, objects.ObjectFormatError):
+        except (FileNotFoundError, ObjectFormatError):
             msg = (
                 "Missing cache for directory '{}'. "
                 "Cache for files inside will be lost. "
@@ -551,14 +554,14 @@ class BaseOutput:
 
         path = str(self.path_info)
         filter_path = str(filter_info) if filter_info else None
-        for entry_key, entry_hash_info in self.dir_cache.items():
+        for entry_key, entry_obj in self.obj:
             entry_path = os.path.join(path, *entry_key)
             if (
                 not filter_path
                 or entry_path == filter_path
                 or entry_path.startswith(filter_path + os.sep)
             ):
-                cache.add(self.scheme, entry_hash_info.value, entry_path)
+                cache.add(self.scheme, entry_obj.hash_info.value, entry_path)
 
         return cache
 
@@ -651,6 +654,8 @@ class BaseOutput:
             )
 
     def merge(self, ancestor, other):
+        from dvc.objects.tree import merge
+
         assert other
 
         if ancestor:
@@ -662,6 +667,6 @@ class BaseOutput:
         self._check_can_merge(self)
         self._check_can_merge(other)
 
-        self.hash_info = objects.merge(
+        self.hash_info = merge(
             self.odb, ancestor_info, self.hash_info, other.hash_info
         )

dvc/remote/base.py

@@ -506,7 +506,7 @@ class Remote:
         return ret
 
     def transfer(self, from_fs, from_info, jobs=None, no_progress_bar=False):
-        from dvc.objects import transfer
+        from dvc.objects.transfer import transfer
 
         return transfer(
             self.odb,

dvc/repo/__init__.py

@@ -463,6 +463,12 @@ class Repo:
         path_parts = os.path.normpath(path).split(os.path.sep)
         return self.DVC_DIR in path_parts
 
+    @cached_property
+    def dvcfs(self):
+        from dvc.fs.dvc import DvcFileSystem
+
+        return DvcFileSystem(self)
+
     @cached_property
     def repo_fs(self):
         from dvc.fs.repo import RepoFileSystem

dvc/repo/add.py

@@ -173,7 +173,7 @@ def _process_stages(
             )
         else:
             from dvc.fs import get_cloud_fs
-            from dvc.objects import transfer
+            from dvc.objects.transfer import transfer
 
             from_fs = get_cloud_fs(repo, url=target)
             out.hash_info = transfer(

dvc/repo/diff.py

@@ -1,5 +1,7 @@
 import logging
 import os
+from collections import defaultdict
+from typing import Dict, List
 
 from dvc.exceptions import PathMissingError
 from dvc.objects.stage import get_file_hash
@@ -130,7 +132,7 @@ def _output_paths(repo, repo_fs, targets):
     def _to_checksum(output):
         if on_working_fs:
             return ostage(
-                repo.odb.local, output.path_info, repo.odb.local.fs,
+                repo.odb.local, output.path_info, repo.odb.local.fs, "md5",
             ).hash_info.value
         return output.hash_info.value
 
@@ -190,29 +192,23 @@ def _targets_to_path_infos(repo_fs, targets):
 
 
 def _calculate_renamed(new, old, added, deleted):
-    old_inverted = {}
+    old_inverted: Dict[str, List[str]] = defaultdict(list)
     # It is needed to be dict of lists to cover cases
     # when repo has paths with same hash
     for path, path_hash in old.items():
-        bucket = old_inverted.get(path_hash, None)
-        if bucket is None:
-            old_inverted[path_hash] = [path]
-        else:
-            bucket.append(path)
+        old_inverted[path_hash].append(path)
 
     renamed = []
     for path in added:
         path_hash = new[path]
-        old_paths = old_inverted.get(path_hash, None)
-        if not old_paths:
-            continue
-        old_path = None
-        for tmp_old_path in old_paths:
-            if tmp_old_path in deleted:
-                old_path = tmp_old_path
-                break
-        if not old_path:
+        old_paths = old_inverted[path_hash]
+        try:
+            iterator = enumerate(old_paths)
+            index = next(idx for idx, path in iterator if path in deleted)
+        except StopIteration:
             continue
+
+        old_path = old_paths.pop(index)
         renamed.append(
             {"path": {"old": old_path, "new": path}, "hash": path_hash}
         )

dvc/repo/experiments/__init__.py

@@ -182,8 +182,6 @@ class Experiments:
                         )
                         self.scm.reset()
 
-                    self._prune_lockfiles()
-
                     # update experiment params from command line
                     if params:
                         self._update_params(params)
@@ -244,34 +242,6 @@ class Experiments:
 
         return stash_rev
 
-    def _prune_lockfiles(self):
-        from dvc.dvcfile import is_lock_file
-
-        # NOTE: dirty DVC lock files must be restored to index state to
-        # avoid checking out incorrect persist or checkpoint outs
-        fs = self.scm.get_fs("HEAD")
-        lock_files = [
-            str(fname)
-            for fname in fs.walk_files(self.scm.root_dir)
-            if is_lock_file(fname)
-        ]
-        if lock_files:
-
-            self.scm.reset(paths=lock_files)
-            self.scm.checkout_index(paths=lock_files, force=True)
-
-    def _prune_untracked_lockfiles(self):
-        from dvc.dvcfile import is_lock_file
-        from dvc.utils.fs import remove
-
-        untracked = [
-            fname
-            for fname in self.scm.untracked_files()
-            if is_lock_file(fname)
-        ]
-        for fname in untracked:
-            remove(fname)
-
     def _stash_msg(
         self,
         rev: str,
@@ -345,9 +315,17 @@ class Experiments:
         queue: bool = False,
         tmp_dir: bool = False,
         checkpoint_resume: Optional[str] = None,
+        reset: bool = False,
         **kwargs,
     ):
         """Reproduce and checkout a single experiment."""
+        if queue and not checkpoint_resume:
+            reset = True
+
+        if reset:
+            self.reset_checkpoints()
+            kwargs["force"] = True
+
         if not (queue or tmp_dir):
             staged, _, _ = self.scm.status()
             if staged:
@@ -370,7 +348,9 @@ class Experiments:
         else:
             checkpoint_resume = self._workspace_resume_rev()
 
-        stash_rev = self.new(checkpoint_resume=checkpoint_resume, **kwargs)
+        stash_rev = self.new(
+            checkpoint_resume=checkpoint_resume, reset=reset, **kwargs
+        )
         if queue:
             logger.info(
                 "Queued experiment '%s' for future execution.", stash_rev[:7],
@@ -709,7 +689,6 @@ class Experiments:
         # result in conflict between workspace params and stashed CLI params).
         self.scm.reset(hard=True)
         with self.scm.detach_head(entry.rev):
-            self._prune_untracked_lockfiles()
             rev = self.stash.pop()
             self.scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
             if entry.branch:
@@ -723,12 +702,17 @@ class Experiments:
                     rev,
                     name=entry.name,
                     rel_cwd=relpath(os.getcwd(), self.scm.root_dir),
+                    log_errors=False,
                 )
 
-                if not exec_result.exp_hash or not exec_result.ref_info:
+                if not exec_result.exp_hash:
                     raise DvcException(
                         f"Failed to reproduce experiment '{rev[:7]}'"
                     )
+                if not exec_result.ref_info:
+                    # repro succeeded but result matches baseline
+                    # (no experiment generated or applied)
+                    return {}
                 exp_rev = self.scm.get_ref(str(exec_result.ref_info))
                 self.scm.set_ref(EXEC_APPLY, exp_rev)
                 return {exp_rev: exec_result.exp_hash}

dvc/repo/experiments/diff.py

@@ -13,12 +13,14 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
         return {}
 
     if a_rev:
-        old = _collect_experiment_commit(repo, a_rev)
+        rev = repo.scm.resolve_rev(a_rev)
+        old = _collect_experiment_commit(repo, rev)
     else:
         old = _collect_experiment_commit(repo, "HEAD")
 
     if b_rev:
-        new = _collect_experiment_commit(repo, b_rev)
+        rev = repo.scm.resolve_rev(b_rev)
+        new = _collect_experiment_commit(repo, rev)
     else:
         new = _collect_experiment_commit(repo, "workspace")
 

dvc/repo/experiments/executor/base.py

@@ -181,16 +181,16 @@ class BaseExecutor(ABC):
                 refs.append(ref)
 
         def on_diverged_ref(orig_ref: str, new_rev: str):
-            orig_rev = dest_scm.get_ref(orig_ref)
-            if dest_scm.diff(orig_rev, new_rev):
-                if force:
-                    logger.debug(
-                        "Replacing existing experiment '%s'", orig_ref,
-                    )
-                    return True
-                if on_diverged:
-                    checkpoint = self.scm.get_ref(EXEC_CHECKPOINT) is not None
-                    on_diverged(orig_ref, checkpoint)
+            if force:
+                logger.debug(
+                    "Replacing existing experiment '%s'", orig_ref,
+                )
+                return True
+
+            checkpoint = self.scm.get_ref(EXEC_CHECKPOINT) is not None
+            self._raise_ref_conflict(dest_scm, orig_ref, new_rev, checkpoint)
+            if on_diverged:
+                on_diverged(orig_ref, checkpoint)
             logger.debug("Reproduced existing experiment '%s'", orig_ref)
             return False
 
@@ -218,6 +218,7 @@ class BaseExecutor(ABC):
         queue: Optional["Queue"] = None,
         rel_cwd: Optional[str] = None,
         name: Optional[str] = None,
+        log_errors: bool = True,
         log_level: Optional[int] = None,
     ) -> "ExecutorResult":
         """Run dvc repro and return the result.
@@ -234,7 +235,7 @@ class BaseExecutor(ABC):
 
         if queue is not None:
             queue.put((rev, os.getpid()))
-        if log_level is not None:
+        if log_errors and log_level is not None:
             cls._set_log_level(log_level)
 
         def filter_pipeline(stages):
@@ -246,7 +247,7 @@ class BaseExecutor(ABC):
         exp_ref: Optional["ExpRefInfo"] = None
         repro_force: bool = False
 
-        with cls._repro_dvc(dvc_dir, rel_cwd) as dvc:
+        with cls._repro_dvc(dvc_dir, rel_cwd, log_errors) as dvc:
             args, kwargs = cls._repro_args(dvc)
             if args:
                 targets: Optional[Union[list, str]] = args[0]
@@ -258,17 +259,19 @@ class BaseExecutor(ABC):
                 "Executor repro with force = '%s'", str(repro_force)
             )
 
-            # NOTE: for checkpoint experiments we handle persist outs slightly
-            # differently than normal:
+            # NOTE: checkpoint outs are handled as a special type of persist
+            # out:
             #
             # - checkpoint out may not yet exist if this is the first time this
             #   experiment has been run, this is not an error condition for
             #   experiments
-            # - at the start of a repro run, we need to remove the persist out
-            #   and restore it to its last known (committed) state (which may
-            #   be removed/does not yet exist) so that our executor workspace
-            #   is not polluted with the (persistent) out from an unrelated
-            #   experiment run
+            # - if experiment was run with --reset, the checkpoint out will be
+            #   removed at the start of the experiment (regardless of any
+            #   dvc.lock entry for the checkpoint out)
+            # - if run without --reset, the checkpoint out will be checked out
+            #   using any hash present in dvc.lock (or removed if no entry
+            #   exists in dvc.lock)
+            checkpoint_reset = kwargs.pop("reset", False)
             dvc_checkout(
                 dvc,
                 targets=targets,
@@ -276,6 +279,7 @@ class BaseExecutor(ABC):
                 force=True,
                 quiet=True,
                 allow_missing=True,
+                checkpoint_reset=checkpoint_reset,
             )
 
             checkpoint_func = partial(
@@ -321,7 +325,9 @@ class BaseExecutor(ABC):
 
     @classmethod
     @contextmanager
-    def _repro_dvc(cls, dvc_dir: Optional[str], rel_cwd: Optional[str]):
+    def _repro_dvc(
+        cls, dvc_dir: Optional[str], rel_cwd: Optional[str], log_errors: bool
+    ):
         from dvc.repo import Repo
 
         dvc = Repo(dvc_dir)
@@ -342,10 +348,12 @@ class BaseExecutor(ABC):
         except CheckpointKilledError:
             raise
         except DvcException:
-            logger.exception("")
+            if log_errors:
+                logger.exception("")
             raise
         except Exception:
-            logger.exception("unexpected error")
+            if log_errors:
+                logger.exception("unexpected error")
             raise
         finally:
             dvc.close()
@@ -358,6 +366,8 @@ class BaseExecutor(ABC):
         if os.path.exists(args_path):
             args, kwargs = cls.unpack_repro_args(args_path)
             remove(args_path)
+            # explicitly git rm/unstage the args file
+            dvc.scm.add([args_path])
         else:
             args = []
             kwargs = {}
@@ -396,6 +406,7 @@ class BaseExecutor(ABC):
             logger.debug("No changes to commit")
             raise UnchangedExperimentError(rev)
 
+        check_conflict = False
         branch = scm.get_ref(EXEC_BRANCH, follow=False)
         if branch:
             old_ref = rev
@@ -406,21 +417,41 @@ class BaseExecutor(ABC):
             ref_info = ExpRefInfo(baseline_rev, name)
             branch = str(ref_info)
             old_ref = None
-            if not force and scm.get_ref(branch):
-                if checkpoint:
-                    raise CheckpointExistsError(ref_info.name)
-                raise ExperimentExistsError(ref_info.name)
-            logger.debug("Commit to new experiment branch '%s'", branch)
+            if scm.get_ref(branch):
+                if not force:
+                    check_conflict = True
+                logger.debug(
+                    "%s existing experiment branch '%s'",
+                    "Replace" if force else "Reuse",
+                    branch,
+                )
+            else:
+                logger.debug("Commit to new experiment branch '%s'", branch)
 
         scm.add([], update=True)
         scm.commit(f"dvc: commit experiment {exp_hash}", no_verify=True)
         new_rev = scm.get_rev()
-        scm.set_ref(branch, new_rev, old_ref=old_ref)
+        if check_conflict:
+            new_rev = cls._raise_ref_conflict(scm, branch, new_rev, checkpoint)
+        else:
+            scm.set_ref(branch, new_rev, old_ref=old_ref)
         scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
         if checkpoint:
             scm.set_ref(EXEC_CHECKPOINT, new_rev)
         return new_rev
 
+    @staticmethod
+    def _raise_ref_conflict(scm, ref, new_rev, checkpoint):
+        # If this commit is a duplicate of the existing commit at 'ref', return
+        # the existing commit. Otherwise, error out and require user to re-run
+        # with --force as needed
+        orig_rev = scm.get_ref(ref)
+        if scm.diff(orig_rev, new_rev):
+            if checkpoint:
+                raise CheckpointExistsError(ref)
+            raise ExperimentExistsError(ref)
+        return orig_rev
+
     @staticmethod
     def _set_log_level(level):
         from dvc.logger import disable_other_loggers

dvc/repo/experiments/executor/local.py

@@ -3,13 +3,29 @@ import os
 from tempfile import TemporaryDirectory
 from typing import Optional
 
-from dvc.utils.fs import remove
-
 from .base import BaseExecutor
 
 logger = logging.getLogger(__name__)
 
 
+class ExpTemporaryDirectory(TemporaryDirectory):
+    # Python's TemporaryDirectory cleanup shutil.rmtree wrapper does not handle
+    # git read-only dirs cleanly in Windows on Python <3.8, so we use our own
+    # remove(). See:
+    # https://github.com/iterative/dvc/pull/5425
+    # https://bugs.python.org/issue26660
+
+    @classmethod
+    def _rmtree(cls, name):
+        from dvc.utils.fs import remove
+
+        remove(name)
+
+    def cleanup(self):
+        if self._finalizer.detach():
+            self._rmtree(self.name)
+
+
 class BaseLocalExecutor(BaseExecutor):
     """Base local machine executor."""
 
@@ -37,7 +53,7 @@ class TempDirExecutor(BaseLocalExecutor):
         cache_dir: Optional[str] = None,
         **kwargs,
     ):
-        self._tmp_dir = TemporaryDirectory(dir=tmp_dir)
+        self._tmp_dir = ExpTemporaryDirectory(dir=tmp_dir)
         kwargs["root_dir"] = self._tmp_dir.name
         super().__init__(*args, **kwargs)
         if cache_dir:
@@ -55,4 +71,4 @@ class TempDirExecutor(BaseLocalExecutor):
     def cleanup(self):
         super().cleanup()
         logger.debug("Removing tmpdir '%s'", self._tmp_dir)
-        remove(self._tmp_dir.name)
+        self._tmp_dir.cleanup()

dvc/repo/experiments/run.py

@@ -15,7 +15,6 @@ def run(
     run_all: bool = False,
     jobs: int = 1,
     tmp_dir: bool = False,
-    reset: bool = False,
     **kwargs,
 ) -> dict:
     """Reproduce the specified targets as an experiment.
@@ -25,10 +24,6 @@ def run(
     Returns a dict mapping new experiment SHAs to the results
     of `repro` for that experiment.
     """
-    if reset:
-        repo.experiments.reset_checkpoints()
-        kwargs["force"] = True
-
     if run_all:
         return repo.experiments.reproduce_queued(jobs=jobs)
 

dvc/repo/fetch.py

@@ -115,6 +115,7 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
                     odb,
                     path_info,
                     repo.repo_fs,
+                    "md5",
                     jobs=jobs,
                     follow_subrepos=False,
                 )

dvc/repo/reproduce.py

@@ -61,7 +61,11 @@ def _track_stage(stage):
 
     stage.repo.scm.track_file(stage.dvcfile.relpath)
     for dep in stage.deps:
-        if not dep.use_scm_ignore and dep.is_in_repo:
+        if (
+            not dep.use_scm_ignore
+            and dep.is_in_repo
+            and not stage.repo.repo_fs.isdvc(dep.path_info)
+        ):
             stage.repo.scm.track_file(relpath(dep.path_info))
     for out in stage.outs:
         if not out.use_scm_ignore and out.is_in_repo:

dvc/scm/git/__init__.py

@@ -343,7 +343,7 @@ class Git(Base):
         from .objects import GitTrie
 
         resolved = self.resolve_rev(rev)
-        tree_obj = self._backend_func("get_tree_obj", rev=resolved)
+        tree_obj = self.pygit2.get_tree_obj(rev=resolved)
         trie = GitTrie(tree_obj, resolved)
         return GitFileSystem(self.root_dir, trie, **kwargs)
 
@@ -377,7 +377,7 @@ class Git(Base):
     _stash_iter = partialmethod(_backend_func, "_stash_iter")
     _stash_push = partialmethod(_backend_func, "_stash_push")
     _stash_apply = partialmethod(_backend_func, "_stash_apply")
-    reflog_delete = partialmethod(_backend_func, "reflog_delete")
+    _stash_drop = partialmethod(_backend_func, "_stash_drop")
     describe = partialmethod(_backend_func, "describe")
     diff = partialmethod(_backend_func, "diff")
     reset = partialmethod(_backend_func, "reset")

dvc/scm/git/backend/base.py

@@ -67,7 +67,11 @@ class BaseGitBackend(ABC):
 
     @abstractmethod
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs,
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
         pass
 
@@ -96,7 +100,7 @@ class BaseGitBackend(ABC):
         pass
 
     @abstractmethod
-    def is_dirty(self, **kwargs) -> bool:
+    def is_dirty(self, untracked_files: bool = False) -> bool:
         pass
 
     @abstractmethod
@@ -245,17 +249,21 @@ class BaseGitBackend(ABC):
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
-        """Push a commit onto the specified stash."""
+        """Push a commit onto the specified stash.
+
+        Returns a tuple of the form (rev, need_reset) where need_reset
+        indicates whether or not the workspace should be `reset --hard`
+        (some backends will not clean the workspace after creating a stash
+        commit).
+        """
 
     @abstractmethod
     def _stash_apply(self, rev: str):
         """Apply the specified stash revision."""
 
     @abstractmethod
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        """Delete the specified reflog entry."""
+    def _stash_drop(self, ref: str, index: int):
+        """Drop the specified stash revision."""
 
     @abstractmethod
     def describe(

dvc/scm/git/backend/dulwich.py

@@ -189,7 +189,11 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
             raise SCMError("Git commit failed") from exc
 
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs,
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
         raise NotImplementedError
 
@@ -222,8 +226,9 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
                 return True
         return False
 
-    def is_dirty(self, **kwargs) -> bool:
-        raise NotImplementedError
+    def is_dirty(self, untracked_files: bool = False) -> bool:
+        staged, unstaged, untracked = self.status()
+        return bool(staged or unstaged or (untracked_files and untracked))
 
     def active_branch(self) -> str:
         raise NotImplementedError
@@ -244,7 +249,10 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
         return DulwichObject(self.repo, ".", stat.S_IFDIR, tree.id)
 
     def get_rev(self) -> str:
-        raise NotImplementedError
+        rev = self.get_ref("HEAD")
+        if rev:
+            return rev
+        raise SCMError("Empty git repo")
 
     def resolve_rev(self, rev: str) -> str:
         raise NotImplementedError
@@ -535,10 +543,17 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def _stash_apply(self, rev: str):
         raise NotImplementedError
 
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        raise NotImplementedError
+    def _stash_drop(self, ref: str, index: int):
+        from dvc.scm.git import Stash
+
+        if ref == Stash.DEFAULT_STASH:
+            raise NotImplementedError
+
+        stash = self._get_stash(ref)
+        try:
+            stash.drop(index)
+        except ValueError as exc:
+            raise SCMError("Failed to drop stash entry") from exc
 
     def describe(
         self,

dvc/scm/git/backend/gitpython.py

@@ -217,12 +217,16 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
             raise SCMError("Git pre-commit hook failed") from exc
 
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
         if create_new:
-            self.repo.git.checkout("HEAD", b=branch, **kwargs)
+            self.repo.git.checkout("HEAD", b=branch, force=force, **kwargs)
         else:
-            self.repo.git.checkout(branch, **kwargs)
+            self.repo.git.checkout(branch, force=force, **kwargs)
 
     def pull(self, **kwargs):
         infos = self.repo.remote().pull(**kwargs)
@@ -249,8 +253,8 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def is_tracked(self, path):
         return bool(self.repo.git.ls_files(path))
 
-    def is_dirty(self, **kwargs):
-        return self.repo.is_dirty(**kwargs)
+    def is_dirty(self, untracked_files: bool = False) -> bool:
+        return self.repo.is_dirty(untracked_files=untracked_files)
 
     def active_branch(self):
         return self.repo.active_branch.name
@@ -495,16 +499,22 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
                 ) from exc
             raise SCMError("Could not apply stash") from exc
 
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        args = ["delete"]
-        if updateref:
-            args.append("--updateref")
-        if rewrite:
-            args.append("--rewrite")
-        args.append(ref)
-        self.git.reflog(*args)
+    def _stash_drop(self, ref: str, index: int):
+        from git.exc import GitCommandError
+
+        from dvc.scm.git import Stash
+
+        if ref == Stash.DEFAULT_STASH:
+            self.git.stash("drop", index)
+            return
+
+        self.git.reflog(
+            "delete", "--updateref", "--rewrite", f"{ref}@{{{index}}}"
+        )
+        try:
+            self.git.reflog("exists", ref)
+        except GitCommandError:
+            self.remove_ref(ref)
 
     def describe(
         self,

dvc/scm/git/backend/pygit2.py

@@ -1,9 +1,12 @@
 import locale
 import logging
 import os
+import stat
 from io import BytesIO, StringIO
 from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
 
+from funcy import cached_property
+
 from dvc.scm.base import MergeConflictError, RevError, SCMError
 from dvc.utils import relpath
 
@@ -31,6 +34,8 @@ class Pygit2Object(GitObject):
 
     @property
     def mode(self):
+        if not self.obj.filemode and self.obj.type_str == "tree":
+            return stat.S_IFDIR
         return self.obj.filemode
 
     def scandir(self) -> Iterable["Pygit2Object"]:
@@ -61,12 +66,20 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         self._stashes: dict = {}
 
     def close(self):
+        if hasattr(self, "_refdb"):
+            del self._refdb
         self.repo.free()
 
     @property
     def root_dir(self) -> str:
         return self.repo.workdir
 
+    @cached_property
+    def _refdb(self):
+        from pygit2 import RefdbFsBackend
+
+        return RefdbFsBackend(self.repo)
+
     @staticmethod
     def clone(
         url: str,
@@ -87,14 +100,20 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         raise NotImplementedError
 
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs,
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
-        from pygit2 import GitError
+        from pygit2 import GIT_CHECKOUT_FORCE, GitError
+
+        checkout_strategy = GIT_CHECKOUT_FORCE if force else None
 
         if create_new:
             commit = self.repo.revparse_single("HEAD")
             new_branch = self.repo.branches.local.create(branch, commit)
-            self.repo.checkout(new_branch)
+            self.repo.checkout(new_branch, strategy=checkout_strategy)
         else:
             if branch == "-":
                 branch = "@{-1}"
@@ -102,7 +121,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
                 commit, ref = self.repo.resolve_refish(branch)
             except (KeyError, GitError):
                 raise RevError(f"unknown Git revision '{branch}'")
-            self.repo.checkout_tree(commit)
+            self.repo.checkout_tree(commit, strategy=checkout_strategy)
             detach = kwargs.get("detach", False)
             if ref and not detach:
                 self.repo.set_head(ref.name)
@@ -133,7 +152,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
     def is_tracked(self, path: str) -> bool:
         raise NotImplementedError
 
-    def is_dirty(self, **kwargs) -> bool:
+    def is_dirty(self, untracked_files: bool = False) -> bool:
         raise NotImplementedError
 
     def active_branch(self) -> str:
@@ -210,12 +229,16 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         if old_ref and old_ref != self.get_ref(name, follow=False):
             raise SCMError(f"Failed to set '{name}'")
 
-        if symbolic:
-            ref = self.repo.create_reference_symbolic(name, new_ref, True)
-        else:
-            ref = self.repo.create_reference_direct(name, new_ref, True)
         if message:
-            ref.set_target(new_ref, message)
+            self._refdb.ensure_log(name)
+        if symbolic:
+            self.repo.create_reference_symbolic(
+                name, new_ref, True, message=message
+            )
+        else:
+            self.repo.create_reference_direct(
+                name, new_ref, True, message=message
+            )
 
     def get_ref(self, name, follow: bool = True) -> Optional[str]:
         from pygit2 import GIT_REF_SYMBOLIC
@@ -302,15 +325,64 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
-        raise NotImplementedError
+        from dvc.scm.git import Stash
+
+        oid = self.repo.stash(
+            self.repo.default_signature,
+            message=message,
+            include_untracked=include_untracked,
+        )
+        commit = self.repo[oid]
+
+        if ref != Stash.DEFAULT_STASH:
+            self.set_ref(ref, commit.id, message=commit.message)
+            self.repo.stash_drop()
+        return str(oid), False
 
     def _stash_apply(self, rev: str):
-        raise NotImplementedError
+        from pygit2 import (
+            GIT_CHECKOUT_RECREATE_MISSING,
+            GIT_CHECKOUT_SAFE,
+            GitError,
+        )
 
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        raise NotImplementedError
+        from dvc.scm.git import Stash
+
+        def _apply(index):
+            try:
+                self.repo.index.read(False)
+                self.repo.stash_apply(
+                    index,
+                    strategy=GIT_CHECKOUT_SAFE | GIT_CHECKOUT_RECREATE_MISSING,
+                )
+            except GitError as exc:
+                raise MergeConflictError(
+                    "Stash apply resulted in merge conflicts"
+                ) from exc
+
+        # libgit2 stash apply only accepts refs/stash items by index. If rev is
+        # not in refs/stash, we will push it onto the stash, and then pop it
+        commit, _ref = self.repo.resolve_refish(rev)
+        stash = self.repo.references.get(Stash.DEFAULT_STASH)
+        if stash:
+            for i, entry in enumerate(stash.log()):
+                if entry.oid_new == commit.id:
+                    _apply(i)
+                    return
+
+        self.set_ref(Stash.DEFAULT_STASH, commit.id, message=commit.message)
+        try:
+            _apply(0)
+        finally:
+            self.repo.stash_drop()
+
+    def _stash_drop(self, ref: str, index: int):
+        from dvc.scm.git import Stash
+
+        if ref != Stash.DEFAULT_STASH:
+            raise NotImplementedError
+
+        self.repo.stash_drop(index)
 
     def describe(
         self,

dvc/scm/git/stash.py

@@ -1,11 +1,9 @@
 """git stash convenience wrapper."""
 
 import logging
-import os
 from typing import Optional
 
 from dvc.scm.base import SCMError
-from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -62,19 +60,12 @@ class Stash:
         self.scm._stash_apply(rev)  # pylint: disable=protected-access
 
     def drop(self, index: int = 0):
-        ref = "{0}@{{{1}}}".format(self.ref, index)
         if index < 0 or index >= len(self):
-            raise SCMError(f"Invalid stash ref '{ref}'")
-        logger.debug("Dropping '%s'", ref)
-        self.scm.reflog_delete(ref, updateref=True, rewrite=True)
-
-        # if we removed the last reflog entry, delete the ref and reflog
-        if len(self) == 0:
-            self.scm.remove_ref(self.ref)
-            parts = self.ref.split("/")
-            reflog = os.path.join(self.scm.root_dir, ".git", "logs", *parts)
-            if os.path.exists(reflog):
-                remove(reflog)
+            raise SCMError(f"Invalid stash ref '{self.ref}@{{{index}}}'")
+        logger.debug("Dropping '%s@{%d}'", self.ref, index)
+        self.scm._stash_drop(  # pylint: disable=protected-access
+            self.ref, index
+        )
 
     def clear(self):
         logger.debug("Clear stash '%s'", self.ref)

dvc/stage/imports.py

@@ -12,7 +12,7 @@ def _update_import_on_remote(stage, remote, jobs):
             "be updated with --to-remote"
         )
 
-    url = stage.deps[0].path_info.url
+    url = stage.deps[0].def_path
     stage.outs[0].hash_info = stage.repo.cloud.transfer(
         url, jobs=jobs, remote=remote, command="update"
     )

dvc/state.py

@@ -76,7 +76,10 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         if not tmp_dir:
             return
 
-        config = {"eviction_policy": "least-recently-used"}
+        config = {
+            "eviction_policy": "least-recently-used",
+            "disk_pickle_protocol": 4,
+        }
         self.links = Cache(directory=os.path.join(tmp_dir, "links"), **config)
         self.md5s = Cache(directory=os.path.join(tmp_dir, "md5s"), **config)
 

dvc/ui/__init__.py

@@ -7,9 +7,6 @@ from funcy import cached_property
 from dvc.progress import Tqdm
 from dvc.utils import colorize
 
-NEWLINE = "\n"
-SEP = " "
-
 
 class Formatter:
     def __init__(self, theme: Dict = None, defaults: Dict = None) -> None:
@@ -30,14 +27,17 @@ class Console:
         formatter: Formatter = None,
         output: TextIO = None,
         error: TextIO = None,
-        disable: bool = False,
+        enable: bool = False,
     ) -> None:
         self._input: TextIO = sys.stdin
         self._output: TextIO = output or sys.stdout
         self._error: TextIO = error or sys.stderr
 
         self.formatter: Formatter = formatter or Formatter()
-        self.disabled: bool = disable
+        self._enabled: bool = enable
+
+    def enable(self):
+        self._enabled = True
 
     def success(self, message: str) -> None:
         self.write(message, style="success")
@@ -52,8 +52,8 @@ class Console:
         self,
         *objects: Any,
         style: str = None,
-        sep: str = SEP,
-        end: str = NEWLINE,
+        sep: str = None,
+        end: str = None,
         flush: bool = False,
     ) -> None:
         return self.write(
@@ -69,12 +69,12 @@ class Console:
         self,
         *objects: Any,
         style: str = None,
-        sep: str = SEP,
-        end: str = NEWLINE,
+        sep: str = None,
+        end: str = None,
         file: TextIO = None,
         flush: bool = False,
     ) -> None:
-        if self.disabled:
+        if not self._enabled:
             return
 
         file = file or self._output
@@ -152,9 +152,11 @@ class Console:
         self.write(ret)
 
 
-if __name__ == "__main__":
-    ui = Console()
+ui = Console()
 
+
+if __name__ == "__main__":
+    ui.enable()
     ui.write("No default remote set")
     ui.success("Everything is up to date.")
     ui.warn("Run queued experiments will be removed.")

dvc/utils/fs.py

@@ -180,7 +180,10 @@ def makedirs(path, exist_ok=False, mode=None):
         if not exist_ok or not os.path.isdir(path):
             raise
 
-    os.chmod(path, mode)
+    try:
+        os.chmod(path, mode)
+    except OSError:
+        logger.trace("failed to chmod '%o' '%s'", mode, path, exc_info=True)
 
 
 def copyfile(src, dest, no_progress_bar=False, name=None):

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.0.1"
+_BASE_VERSION = "2.0.11"
 
 
 def _generate_version(base_version):

scripts/build-requirements.txt

@@ -1,2 +1,2 @@
-PyInstaller==3.5
+PyInstaller==4.2
 numpy==1.19.3

scripts/build.py

@@ -0,0 +1,103 @@
+import argparse
+import importlib
+import pathlib
+import textwrap
+from subprocess import STDOUT, check_call
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path.parent / "dvc"
+pyinstaller = path / "pyinstaller"
+innosetup = path / "innosetup"
+fpm = path / "fpm"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type"
+)
+parser.add_argument("--sign-application", default=False, action="store_true")
+parser.add_argument("--application-id")
+parser.add_argument("--sign-installer", default=False, action="store_true")
+parser.add_argument("--installer-id")
+parser.add_argument("--notarize")
+parser.add_argument("--apple-id-username")
+parser.add_argument("--apple-id-password")
+args = parser.parse_args()
+
+(dvc / "utils" / "build.py").write_text(f'PKG = "{args.pkg}"')
+
+# Autogenerate version, similar to what we do in setup.py
+spec = importlib.util.spec_from_file_location(
+    "dvc.version", dvc / "version.py"
+)
+dvc_version = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(dvc_version)
+version = dvc_version.__version__
+
+(dvc / "version.py").write_text(
+    textwrap.dedent(
+        f"""\
+        # AUTOGENERATED at build time by scripts/build.py
+        __version__ = "{version}"
+    """
+    )
+)
+
+check_call(
+    ["python", "build.py"], cwd=pyinstaller, stderr=STDOUT,
+)
+
+if args.sign_application:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not args.application_id:
+        print("--sign-application requires --application-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--application-id", args.application_id],
+        cwd=pyinstaller,
+        stderr=STDOUT,
+    )
+
+if args.pkg == "exe":
+    check_call(
+        ["python", "build.py"], cwd=innosetup, stderr=STDOUT,
+    )
+else:
+    check_call(
+        ["python", "build.py", args.pkg], cwd=fpm, stderr=STDOUT,
+    )
+
+if args.sign_installer:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all(
+        [args.installer_id, args.apple_id_username, args.apple_id_password]
+    ):
+        print("--sign-installer requires --installer-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--installer-id", args.installer_id],
+        cwd=fpm,
+        stderr=STDOUT,
+    )
+
+if args.notarize:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all([args.apple_id_username, args.apple_id_password]):
+        print(
+            "--notarize requires --apple-id-username and --apple-id-password"
+        )
+        exit(1)
+    check_call(
+        [
+            "python",
+            "notarize.py",
+            "--apple-id-username",
+            args.apple_id_username,
+            "--apple-id-password",
+            args.apple_id_password,
+        ],
+        cwd=fpm,
+        stderr=STDOUT,
+    )

scripts/build_posix.sh

@@ -1,139 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-FPM_FLAGS='--description "Data Version Control - datasets, models, and experiments versioning for ML or data science projects"'
-FPM_FLAGS+=' -n dvc -s dir -f'
-FPM_FLAGS+=' --url https://dvc.org'
-FPM_FLAGS+=' --license "Apache License 2.0"'
-
-if [[ "$(uname)" == 'Linux' ]]; then
-  INSTALL_DIR=usr
-  FPM_FLAGS+=' --depends "git >= 1.7.0"'  # needed for gitpython
-else
-  INSTALL_DIR=usr/local
-  FPM_FLAGS+=' --osxpkg-identifier-prefix com.iterative'
-  FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh'
-  FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh'
-fi
-
-BUILD_DIR=build
-BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin
-LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
-
-FPM_PACKAGE_DIRS="usr"
-ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc
-if [[ "$(uname)" == 'Linux' ]]; then
-  BASH_CMPLT_DIR=etc/bash_completion.d
-  FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
-else
-  BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
-fi
-
-print_error() {
-  echo -e "\e[31m$1\e[0m" >&2
-}
-
-if [ ! -d "dvc" ]; then
-  print_error "Please run this script from repository root"
-  exit 1
-fi
-
-trap 'print_error "FAIL"; exit 1' ERR
-
-print_info() {
-  echo -e "\e[32m$1\e[0m"
-}
-
-command_exists() {
-  command -v $1 >/dev/null 2>&1
-}
-
-fpm_build() {
-  print_info "Building $1..."
-  VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
-  eval "fpm -t $1 $FPM_FLAGS -v $VERSION -C $BUILD_DIR $FPM_PACKAGE_DIRS"
-}
-
-cleanup() {
-  print_info "Cleaning up..."
-  rm -rf build
-}
-
-install_dependencies() {
-  print_info "Installing fpm..."
-  if command_exists dnf; then
-    sudo dnf install ruby-devel gcc make rpm-build
-  elif command_exists yum; then
-    sudo yum install ruby-devel gcc make rpm-build
-  elif command_exists apt-get; then
-    sudo apt-get update -y
-    sudo apt-get install ruby-dev build-essential rpm python-pip python-dev
-  elif command_exists brew; then
-    brew install ruby
-  else
-    echo "Unable to install fpm dependencies" && exit 1
-  fi
-
-  gem install --no-document fpm
-
-  # https://github.com/iterative/dvc/issues/2970
-  gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler
-
-  print_info "Installing requirements..."
-  pip install .[all]
-  pip install -r scripts/build-requirements.txt
-}
-
-build_dvc() {
-  print_info "Building dvc binary..."
-  pyinstaller \
-    --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \
-    --name dvc \
-    --distpath $LIB_DIR \
-    --specpath $BUILD_DIR
-
-  $LIB_DIR/dvc/dvc --help
-
-  # NOTE: in osxpkg fpm replaces symlinks with actual file that it
-  # points to, so we need to use after-install hook. See FPM_FLAGS
-  # above.
-  if [[ "$(uname)" == 'Linux' ]]; then
-    mkdir -p $BIN_DIR
-    pushd $BIN_DIR
-    ln -s ../lib/dvc/dvc dvc
-    popd
-    $BIN_DIR/dvc --help
-  fi
-
-  # NOTE: temporarily not adding scripts to mac package. See [1]
-  # [1] https://github.com/iterative/dvc/issues/2585
-  if [[ "$(uname)" == 'Linux' ]]; then
-    mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
-    $LIB_DIR/dvc/dvc completion -s bash > $BUILD_DIR/$BASH_CMPLT_DIR/dvc
-
-    mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
-    $LIB_DIR/dvc/dvc completion -s zsh > $BUILD_DIR/$ZSH_CMPLT_DIR/_dvc
-  fi
-}
-
-build() {
-  cleanup
-  echo "PKG = \"$1\"" >dvc/utils/build.py
-  build_dvc
-  fpm_build $1
-}
-
-install_dependencies
-
-if [[ "$(uname)" == 'Linux' ]]; then
-  build rpm
-  build deb
-else
-  build osxpkg
-fi
-
-cleanup
-
-print_info "Successfully built dvc packages"

scripts/build_windows.cmd

@@ -1,48 +0,0 @@
-@echo Off
-
-echo ====== Starting to build dvc installer for Windows... ======
-
-if not exist dvc\NUL (echo Error: Please run this script from repository root && goto :error)
-
-rmdir /Q /S build
-rmdir /Q /S dist
-del /Q /S dvc.spec
-del /Q /S "dvc-*.exe"
-
-where pip
-if %errorlevel% neq 0 (echo Error: pip not found && goto :error)
-
-where choco
-if %errorlevel% neq 0 (echo Error: choco not found && goto :error)
-
-choco install InnoSetup
-call refreshenv
-where iscc
-if %errorlevel% neq 0 (echo Error: Couldn't find Inno Setup compiler. && goto :error)
-
-echo ====== Installing requirements... ======
-echo PKG = "exe" > dvc\utils\build.py
-call pip install .[all] || goto :error
-call pip install -r scripts\build-requirements.txt || goto :error
-call dvc pull || goto :error
-
-echo ====== Building dvc binary... ======
-call pyinstaller --additional-hooks-dir scripts\hooks dvc/__main__.py --name dvc --specpath build
-
-echo ====== Testing dvc binary... ======
-call dist\dvc\dvc.exe version || goto :error
-
-echo ====== Copying additional files... ======
-copy scripts\innosetup\addSymLinkPermissions.ps1 dist\ || goto :error
-
-echo ====== Building dvc installer... ======
-set PYTHONPATH=%cd%
-call python scripts\innosetup\config_gen.py || goto :error
-call iscc scripts\innosetup\setup.iss || goto :error
-
-echo ====== DONE ======
-goto :EOF
-
-:error
-echo ====== FAIL ======
-exit /b 1

scripts/fpm/.gitignore

@@ -0,0 +1,4 @@
+/build
+/*.deb
+/*.rpm
+/*.pkg

scripts/fpm/build.py

@@ -0,0 +1,96 @@
+import argparse
+import os
+import pathlib
+import shutil
+from subprocess import STDOUT, check_call, check_output
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path.parent.parent / "dvc"
+pyinstaller = path.parent / "pyinstaller"
+
+build = path / "build"
+install = build / "usr"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "pkg", choices=["deb", "rpm", "osxpkg"], help="package type"
+)
+args = parser.parse_args()
+
+flags = [
+    "--description",
+    '"Data Version Control | Git for Data & Models"',
+    "-n",
+    "dvc",
+    "-s",
+    "dir",
+    "-f",
+    "--license",
+    '"Apache License 2.0"',
+]
+
+if args.pkg == "osxpkg":
+    install /= "local"
+    bash_dir = install / "etc" / "bash_completion.d"
+    dirs = ["usr"]
+    flags.extend(
+        [
+            "--osxpkg-identifier-prefix",
+            "com.iterative",
+            "--after-install",
+            path / "after-install.sh",
+            "--after-remove",
+            path / "after-remove.sh",
+        ]
+    )
+else:
+    bash_dir = build / "etc" / "bash_completion.d"
+    dirs = ["usr", "etc"]
+    flags.extend(["--depends", "git >= 1.7.0"])  # needed for gitpython
+
+try:
+    shutil.rmtree(build)
+except FileNotFoundError:
+    pass
+
+lib = install / "lib"
+lib.mkdir(parents=True)
+shutil.copytree(pyinstaller / "dist" / "dvc", lib / "dvc")
+
+if args.pkg != "osxpkg":
+    # NOTE: in osxpkg fpm replaces symlinks with actual file that it
+    # points to, so we need to use after-install hook.
+    (install / "bin").mkdir()
+    os.symlink("../lib/dvc/dvc", install / "bin" / "dvc")
+
+bash_dir.mkdir(parents=True)
+bash_completion = check_output(
+    [lib / "dvc" / "dvc", "completion", "-s", "bash"], text=True
+)
+(bash_dir / "dvc").write_text(bash_completion)
+
+zsh_dir = install / "share" / "zsh" / "site-funtions"
+zsh_dir.mkdir(parents=True)
+zsh_completion = check_output(
+    [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True
+)
+(zsh_dir / "_dvc").write_text(zsh_completion)
+
+version = check_output([lib / "dvc" / "dvc", "--version"], text=True).strip()
+
+check_call(
+    [
+        "fpm",
+        "--verbose",
+        "-t",
+        args.pkg,
+        *flags,
+        "-v",
+        version,
+        "-C",
+        build,
+        *dirs,
+    ],
+    cwd=path,
+    stderr=STDOUT,
+)

scripts/fpm/notarize.py

@@ -0,0 +1,64 @@
+import argparse
+import json
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "Darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to notarize. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--apple-id-username", required=True, help="Apple ID username."
+)
+parser.add_argument(
+    "--apple-id-password",
+    required=True,
+    help=(
+        "Apple ID app-specific password. Note that this is not a regular "
+        "Apple ID password, so you need to generate one at "
+        "https://appleid.apple.com/account/manage"
+    ),
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+
+config = {
+    "notarize": {
+        "path": os.fspath(pkg),
+        "bundle_id": "com.iterative.dvc",
+        "staple": True,
+    },
+    "apple_id": {
+        "username": args.apple_id_username,
+        "password": args.apple_id_password,
+    },
+}
+
+(path / "config.json").write_text(json.dumps(config))
+
+check_call(
+    ["gon", "config.json"], cwd=path, stderr=STDOUT,
+)

scripts/fpm/sign.py

@@ -0,0 +1,54 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to sign. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--installer-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+unsigned = pkg.with_suffix(".unsigned")
+os.rename(pkg, unsigned)
+check_call(
+    [
+        "productsign",
+        "--sign",
+        args.installer_id,
+        os.fspath(unsigned),
+        os.fspath(pkg),
+    ],
+    stderr=STDOUT,
+)
+
+check_call(
+    ["pkgutil", "--check-signature", os.fspath(pkg)], stderr=STDOUT,
+)

scripts/innosetup/.gitignore

@@ -1,4 +1,6 @@
-dvc.ico
-dvc_left.bmp
-dvc_up.bmp
+/dvc.ico
+/dvc_left.bmp
+/dvc_up.bmp
 /config.ini
+/build
+/dvc*.exe

scripts/innosetup/build.py

@@ -0,0 +1,42 @@
+import argparse
+import configparser
+import os
+import pathlib
+import shutil
+from subprocess import STDOUT, check_call, check_output
+
+path = pathlib.Path(__file__).parent.absolute()
+config = path / "config.ini"
+
+dvc = path.parent.parent / "dvc"
+pyinstaller = path.parent / "pyinstaller"
+
+build = path / "build"
+install = build / "usr"
+
+parser = argparse.ArgumentParser()
+args = parser.parse_args()
+
+try:
+    shutil.rmtree(build)
+except FileNotFoundError:
+    pass
+
+build.mkdir()
+shutil.copytree(pyinstaller / "dist" / "dvc", build / "dvc")
+shutil.copy(path / "addSymLinkPermissions.ps1", build)
+
+version = check_output(
+    [os.fspath(build / "dvc" / "dvc"), "--version"], text=True
+).strip()
+
+cfg = configparser.ConfigParser()
+cfg.add_section("Version")
+cfg.set("Version", "version", version)
+
+with (path / "config.ini").open("w") as fobj:
+    cfg.write(fobj)
+
+check_call(
+    ["iscc", "setup.iss"], cwd=path, stderr=STDOUT,
+)

scripts/innosetup/config_gen.py

@@ -1,11 +0,0 @@
-# This script generates config.ini for setup.iss script
-import configparser
-
-from dvc import __version__
-
-config = configparser.ConfigParser()
-config.add_section("Version")
-config.set("Version", "Version", __version__)
-
-with open("scripts/innosetup/config.ini", "w") as f:
-    config.write(f)

scripts/innosetup/setup.iss

@@ -1,8 +1,8 @@
 #define MyAppName "DVC (Data Version Control)"
-#define MyAppVersion ReadIni(".\scripts\innosetup\config.ini", "Version", "version", "unknown")
+#define MyAppVersion ReadIni(SourcePath + "\config.ini", "Version", "version", "unknown")
 #define MyAppPublisher "Dmitry Petrov"
-#define MyAppURL "https://dataversioncontrol.com/"
-#define MyAppDir "..\..\dist\dvc"
+#define MyAppURL "https://dvc.org"
+#define MyAppDir SourcePath + "\build\dvc"
 
 [Setup]
 AppId={{8258CE8A-110E-4E0D-AE60-FEE00B15F041}
@@ -19,7 +19,7 @@ LicenseFile=..\..\LICENSE
 OutputBaseFilename=dvc-{#MyAppVersion}
 Compression=lzma2/max
 SolidCompression=yes
-OutputDir=..\..\
+OutputDir=.
 ChangesEnvironment=yes
 SetupIconFile=dvc.ico
 WizardSmallImageFile=dvc_up.bmp

scripts/pyinstaller/.gitignore

@@ -0,0 +1,4 @@
+/build
+/dist
+/dvc.spec
+/dvc

scripts/pyinstaller/build.py

@@ -0,0 +1,22 @@
+import os
+import pathlib
+from subprocess import STDOUT, check_call
+
+path = pathlib.Path(__file__).parent.absolute()
+hooks = path / "hooks"
+dvc = path.parent.parent / "dvc"
+entry = dvc / "__main__.py"
+
+check_call(
+    [
+        "pyinstaller",
+        "--additional-hooks-dir",
+        os.fspath(hooks),
+        "--name",
+        "dvc",
+        "-y",
+        os.fspath(entry),
+    ],
+    cwd=path,
+    stderr=STDOUT,
+)

scripts/pyinstaller/entitlements.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<!-- These are required for binaries built by PyInstaller -->
+	<key>com.apple.security.cs.allow-jit</key>
+	<true/>
+	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+	<true/>
+</dict>
+</plist>

scripts/pyinstaller/sign.py

@@ -0,0 +1,37 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--application-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path / "dist" / "dvc"
+for root, _, fnames in os.walk(dvc):
+    for fname in fnames:
+        fpath = os.path.join(root, fname)
+        print(f"signing {fpath}")
+        check_call(
+            [
+                "codesign",
+                "--force",
+                "-s",
+                args.application_id,
+                "-o",
+                "runtime",
+                "--entitlements",
+                "entitlements.plist",
+                fpath,
+            ],
+            stderr=STDOUT,
+        )

setup.py

@@ -50,8 +50,8 @@ install_requires = [
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "gitpython>3",
-    "dulwich>=0.20.19",
-    "pygit2>=1.4.0",
+    "dulwich>=0.20.20",
+    "pygit2>=1.5.0",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
@@ -95,9 +95,9 @@ install_requires = [
 # Extra dependencies for remote integrations
 
 gs = ["gcsfs>=0.7.2"]
-gdrive = ["pydrive2>=1.7.3", "six >= 1.13.0"]
+gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
 s3 = ["boto3>=1.9.201"]
-azure = ["adlfs>=0.6.3", "azure.identity>=1.4.0", "knack"]
+azure = ["adlfs>=0.6.3", "azure-identity>=1.4.0", "knack"]
 # https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["oss2==2.6.1", "pycryptodome<3.9.9"]
 ssh = ["paramiko[invoke]>=2.7.0"]
@@ -118,7 +118,6 @@ tests_requirements = [
     "wheel>=0.31.1",
     # Test requirements:
     "pytest>=6.0.1,<6.2.2",
-    "pytest-cov",
     "pytest-docker>=0.7.2",
     "pytest-timeout>=1.3.3",
     "pytest-cov>=2.6.1",
@@ -145,10 +144,9 @@ tests_requirements = [
     "urllib3<1.26.0",
     "rangehttpserver==1.2.0",
     "beautifulsoup4==4.4.0",
-    "pylint==2.5.3",
-    "astroid<2.5",  # temporary required for pylint, see #5469
-    "pylint-pytest>=0.3.0",
-    "pylint-plugin-utils",
+    "pylint==2.7.2",
+    "pylint-pytest==1.0.3",
+    "pylint-plugin-utils==0.6",
     "wget",
     "filelock",
     "mypy",
@@ -192,6 +190,7 @@ setup(
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
     packages=find_packages(exclude=["tests"]),
     include_package_data=True,

tests/func/experiments/test_checkpoints.py

@@ -88,10 +88,6 @@ def test_reset_checkpoint(
         checkpoint_stage.addressing, name="foo", tmp_dir=not workspace,
     )
 
-    if workspace:
-        scm.reset(hard=True)
-        scm.gitpython.repo.git.clean(force=True)
-
     results = dvc.experiments.run(
         checkpoint_stage.addressing,
         params=["foo=2"],

tests/func/experiments/test_experiments.py

@@ -418,28 +418,6 @@ def test_untracked(tmp_dir, scm, dvc, caplog, workspace):
         assert fobj.read().strip() == "foo: 2"
 
 
-@pytest.mark.parametrize("workspace", [True, False])
-def test_dirty_lockfile(tmp_dir, scm, dvc, exp_stage, workspace):
-    from dvc.dvcfile import LockfileCorruptedError
-
-    tmp_dir.gen("dvc.lock", "foo")
-
-    with pytest.raises(LockfileCorruptedError):
-        dvc.reproduce(exp_stage.addressing)
-
-    results = dvc.experiments.run(
-        exp_stage.addressing, params=["foo=2"], tmp_dir=not workspace
-    )
-    exp = first(results)
-
-    fs = scm.get_fs(exp)
-    with fs.open(tmp_dir / "metrics.yaml") as fobj:
-        assert fobj.read().strip() == "foo: 2"
-
-    if not workspace:
-        assert (tmp_dir / "dvc.lock").read_text() == "foo"
-
-
 def test_packed_args_exists(tmp_dir, scm, dvc, exp_stage, caplog):
     from dvc.repo.experiments.executor.base import BaseExecutor
 

tests/func/params/test_diff.py

@@ -1,7 +1,3 @@
-import os
-
-import pytest
-
 from dvc.utils import relpath
 from dvc.utils.serialize import dump_yaml
 
@@ -18,132 +14,6 @@ def test_diff_no_changes(tmp_dir, scm, dvc):
     assert dvc.params.diff() == {}
 
 
-@pytest.mark.parametrize("commit_last", [True, False])
-def test_diff_add_similar_files(tmp_dir, scm, dvc, commit_last):
-    if commit_last:
-        last_commit_msg = "commit #2"
-        a_rev = "HEAD~1"
-    else:
-        last_commit_msg = None
-        a_rev = "HEAD"
-
-    tmp_dir.dvc_gen(
-        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}},
-        commit="commit #1",
-    )
-    tmp_dir.dvc_gen(
-        {"dir2": {"file": "text1", "subdir": {"file2": "text2"}}},
-        commit=last_commit_msg,
-    )
-    assert dvc.diff(a_rev) == {
-        "added": [
-            {
-                "path": os.path.join("dir2", ""),
-                "hash": "cb58ee07cb01044db229e4d6121a0dfc.dir",
-            },
-            {
-                "path": os.path.join("dir2", "file"),
-                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
-            },
-            {
-                "path": os.path.join("dir2", "subdir", "file2"),
-                "hash": "fe6123a759017e4a2af4a2d19961ed71",
-            },
-        ],
-        "deleted": [],
-        "modified": [],
-        "renamed": [],
-        "not in cache": [],
-    }
-
-
-@pytest.mark.parametrize("commit_last", [True, False])
-def test_diff_rename_folder(tmp_dir, scm, dvc, commit_last):
-    if commit_last:
-        last_commit_msg = "commit #2"
-        a_rev = "HEAD~1"
-    else:
-        last_commit_msg = None
-        a_rev = "HEAD"
-
-    tmp_dir.dvc_gen(
-        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}},
-        commit="commit #1",
-    )
-    (tmp_dir / "dir").replace(tmp_dir / "dir2")
-    tmp_dir.dvc_add("dir2", commit=last_commit_msg)
-    assert dvc.diff(a_rev) == {
-        "added": [],
-        "deleted": [],
-        "modified": [],
-        "renamed": [
-            {
-                "path": {
-                    "old": os.path.join("dir", ""),
-                    "new": os.path.join("dir2", ""),
-                },
-                "hash": "cb58ee07cb01044db229e4d6121a0dfc.dir",
-            },
-            {
-                "path": {
-                    "old": os.path.join("dir", "file"),
-                    "new": os.path.join("dir2", "file"),
-                },
-                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
-            },
-            {
-                "path": {
-                    "old": os.path.join("dir", "subdir", "file2"),
-                    "new": os.path.join("dir2", "subdir", "file2"),
-                },
-                "hash": "fe6123a759017e4a2af4a2d19961ed71",
-            },
-        ],
-        "not in cache": [],
-    }
-
-
-@pytest.mark.parametrize("commit_last", [True, False])
-def test_diff_rename_file(tmp_dir, scm, dvc, commit_last):
-    if commit_last:
-        last_commit_msg = "commit #2"
-        a_rev = "HEAD~1"
-    else:
-        last_commit_msg = None
-        a_rev = "HEAD"
-
-    paths = tmp_dir.gen(
-        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}}
-    )
-    tmp_dir.dvc_add(paths, commit="commit #1")
-    (tmp_dir / "dir" / "file").replace(tmp_dir / "dir" / "subdir" / "file3")
-
-    tmp_dir.dvc_add(paths, commit=last_commit_msg)
-    assert dvc.diff(a_rev) == {
-        "added": [],
-        "deleted": [],
-        "modified": [
-            {
-                "path": os.path.join("dir", ""),
-                "hash": {
-                    "old": "cb58ee07cb01044db229e4d6121a0dfc.dir",
-                    "new": "a4ac9c339aacc60b6a3152e362c319c8.dir",
-                },
-            }
-        ],
-        "renamed": [
-            {
-                "path": {
-                    "old": os.path.join("dir", "file"),
-                    "new": os.path.join("dir", "subdir", "file3"),
-                },
-                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
-            }
-        ],
-        "not in cache": [],
-    }
-
-
 def test_diff(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo: bar")
     dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)

tests/func/plots/test_show.py

@@ -708,7 +708,7 @@ def test_show_from_subdir(tmp_dir, dvc, caplog):
     with subdir.chdir(), caplog.at_level(logging.INFO, "dvc"):
         assert main(["plots", "show", "metric.json"]) == 0
 
-    assert f"file://{str(subdir)}" in caplog.text
+    assert subdir.as_uri() in caplog.text
     assert (subdir / "plots.html").exists()
 
 

tests/func/remote/test_index.py

@@ -32,7 +32,7 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
 def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote):
     foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
     bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
-    baz_hash = bar.dir_cache.trie.get(("baz",))
+    baz_hash = bar.obj.trie.get(("baz",)).hash_info
     dvc.push()
     with remote.index:
         remote.index.clear()
@@ -49,7 +49,7 @@ def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote):
 def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
     foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
     bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
-    baz_hash = bar.dir_cache.trie.get(("baz",))
+    baz_hash = bar.obj.trie.get(("baz",)).hash_info
 
     dvc.push()
     with remote.index:

tests/func/test_add.py

@@ -109,9 +109,10 @@ def test_add_directory(tmp_dir, dvc):
 
     hash_info = stage.outs[0].hash_info
 
-    dir_info = load(dvc.odb.local, hash_info).hash_info.dir_info
-    for path, _ in dir_info.trie.items():
-        assert "\\" not in path
+    obj = load(dvc.odb.local, hash_info)
+    for key, _ in obj:
+        for part in key:
+            assert "\\" not in part
 
 
 class TestAddDirectoryRecursive(TestDvc):
@@ -516,7 +517,7 @@ class TestAddCommit(TestDvc):
 
 def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
     tmp_dir.gen({"data/data": "foo"})
-    get_dir_hash_counter = mocker.spy(dvc_module.objects.stage, "get_dir_hash")
+    counter = mocker.spy(dvc_module.objects.stage, "_get_tree_obj")
     ret = main(["add", "data"])
     assert ret == 0
 
@@ -525,7 +526,7 @@ def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
 
     ret = main(["status"])
     assert ret == 0
-    assert get_dir_hash_counter.mock.call_count == 1
+    assert counter.mock.call_count == 1
 
 
 class TestShouldPlaceStageInDataDirIfRepositoryBelowSymlink(TestDvc):

tests/func/test_checkout.py

@@ -100,11 +100,10 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
 
         # NOTE: modifying cache file for one of the files inside the directory
         # to check if dvc will detect that the cache is corrupted.
-        _, entry_hash = next(
-            load(self.dvc.odb.local, out.hash_info).hash_info.dir_info.items()
-        )
+        obj = load(self.dvc.odb.local, out.hash_info)
+        _, entry_obj = list(obj)[0]
         cache = os.fspath(
-            self.dvc.odb.local.hash_to_path_info(entry_hash.value)
+            self.dvc.odb.local.hash_to_path_info(entry_obj.hash_info.value)
         )
 
         os.chmod(cache, 0o644)

tests/func/test_data_cloud.py

@@ -287,7 +287,7 @@ def test_verify_hashes(
     remove("dir")
     remove(dvc.odb.local.cache_dir)
 
-    hash_spy = mocker.spy(dvc_module.objects.stage, "get_file_hash")
+    hash_spy = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
 
     dvc.pull()
     assert hash_spy.call_count == 0
@@ -537,3 +537,13 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
 
     dvc.fetch("copy-foo-bar")
     assert len(recurse_list_dir(dvc.odb.local.cache_dir)) == 1
+
+
+def test_pull_partial(tmp_dir, dvc, local_remote):
+    tmp_dir.dvc_gen({"foo": {"bar": {"baz": "baz"}, "spam": "spam"}})
+    dvc.push()
+    clean(["foo"], dvc)
+
+    stats = dvc.pull(os.path.join("foo", "bar"))
+    assert stats["fetched"] == 1
+    assert (tmp_dir / "foo").read_text() == {"bar": {"baz": "baz"}}

tests/func/test_diff.py

@@ -423,3 +423,175 @@ def test_targets_single_file_in_dir_with_file(tmp_dir, scm, dvc):
         "not in cache": [],
         "renamed": [],
     }
+
+
+@pytest.mark.parametrize("commit_last", [True, False])
+def test_diff_add_similar_files(tmp_dir, scm, dvc, commit_last):
+    if commit_last:
+        last_commit_msg = "commit #2"
+        a_rev = "HEAD~1"
+    else:
+        last_commit_msg = None
+        a_rev = "HEAD"
+
+    tmp_dir.dvc_gen(
+        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}},
+        commit="commit #1",
+    )
+    tmp_dir.dvc_gen(
+        {"dir2": {"file": "text1", "subdir": {"file2": "text2"}}},
+        commit=last_commit_msg,
+    )
+    assert dvc.diff(a_rev) == {
+        "added": [
+            {
+                "path": os.path.join("dir2", ""),
+                "hash": "cb58ee07cb01044db229e4d6121a0dfc.dir",
+            },
+            {
+                "path": os.path.join("dir2", "file"),
+                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
+            },
+            {
+                "path": os.path.join("dir2", "subdir", "file2"),
+                "hash": "fe6123a759017e4a2af4a2d19961ed71",
+            },
+        ],
+        "deleted": [],
+        "modified": [],
+        "renamed": [],
+        "not in cache": [],
+    }
+
+
+@pytest.mark.parametrize("commit_last", [True, False])
+def test_diff_rename_folder(tmp_dir, scm, dvc, commit_last):
+    if commit_last:
+        last_commit_msg = "commit #2"
+        a_rev = "HEAD~1"
+    else:
+        last_commit_msg = None
+        a_rev = "HEAD"
+
+    tmp_dir.dvc_gen(
+        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}},
+        commit="commit #1",
+    )
+    (tmp_dir / "dir").replace(tmp_dir / "dir2")
+    tmp_dir.dvc_add("dir2", commit=last_commit_msg)
+    assert dvc.diff(a_rev) == {
+        "added": [],
+        "deleted": [],
+        "modified": [],
+        "renamed": [
+            {
+                "path": {
+                    "old": os.path.join("dir", ""),
+                    "new": os.path.join("dir2", ""),
+                },
+                "hash": "cb58ee07cb01044db229e4d6121a0dfc.dir",
+            },
+            {
+                "path": {
+                    "old": os.path.join("dir", "file"),
+                    "new": os.path.join("dir2", "file"),
+                },
+                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
+            },
+            {
+                "path": {
+                    "old": os.path.join("dir", "subdir", "file2"),
+                    "new": os.path.join("dir2", "subdir", "file2"),
+                },
+                "hash": "fe6123a759017e4a2af4a2d19961ed71",
+            },
+        ],
+        "not in cache": [],
+    }
+
+
+@pytest.mark.parametrize("commit_last", [True, False])
+def test_diff_rename_file(tmp_dir, scm, dvc, commit_last):
+    if commit_last:
+        last_commit_msg = "commit #2"
+        a_rev = "HEAD~1"
+    else:
+        last_commit_msg = None
+        a_rev = "HEAD"
+
+    paths = tmp_dir.gen(
+        {"dir": {"file": "text1", "subdir": {"file2": "text2"}}}
+    )
+    tmp_dir.dvc_add(paths, commit="commit #1")
+    (tmp_dir / "dir" / "file").replace(tmp_dir / "dir" / "subdir" / "file3")
+
+    tmp_dir.dvc_add(paths, commit=last_commit_msg)
+    assert dvc.diff(a_rev) == {
+        "added": [],
+        "deleted": [],
+        "modified": [
+            {
+                "path": os.path.join("dir", ""),
+                "hash": {
+                    "old": "cb58ee07cb01044db229e4d6121a0dfc.dir",
+                    "new": "a4ac9c339aacc60b6a3152e362c319c8.dir",
+                },
+            }
+        ],
+        "renamed": [
+            {
+                "path": {
+                    "old": os.path.join("dir", "file"),
+                    "new": os.path.join("dir", "subdir", "file3"),
+                },
+                "hash": "cef7ccd89dacf1ced6f5ec91d759953f",
+            }
+        ],
+        "not in cache": [],
+    }
+
+
+def test_rename_multiple_files_same_hashes(tmp_dir, scm, dvc):
+    """Test diff by renaming >=2 instances of file with same hashes.
+
+    DVC should be able to detect that they are renames, and should not include
+    them in either of the `added` or the `deleted` section.
+    """
+    tmp_dir.dvc_gen(
+        {"dir": {"foo": "foo", "subdir": {"foo": "foo"}}}, commit="commit #1",
+    )
+    remove(tmp_dir / "dir")
+    # changing foo and subdir/foo to bar and subdir/bar respectively
+    tmp_dir.dvc_gen(
+        {"dir": {"bar": "foo", "subdir": {"bar": "foo"}}}, commit="commit #2"
+    )
+    assert dvc.diff("HEAD~") == {
+        "added": [],
+        "deleted": [],
+        "modified": [
+            {
+                "hash": {
+                    "new": "31b36b3ea5f4485e27f10578c47183b0.dir",
+                    "old": "c7684c8b3b0d28cf80d5305e2d856bfc.dir",
+                },
+                "path": os.path.join("dir", ""),
+            }
+        ],
+        "not in cache": [],
+        "renamed": [
+            {
+                "hash": "acbd18db4cc2f85cedef654fccc4a4d8",
+                "path": {
+                    "new": os.path.join("dir", "bar"),
+                    "old": os.path.join("dir", "foo"),
+                },
+            },
+            {
+                "hash": "acbd18db4cc2f85cedef654fccc4a4d8",
+                "path": {
+                    "new": os.path.join("dir", "subdir", "bar"),
+                    "old": os.path.join("dir", "subdir", "foo"),
+                },
+            },
+        ],
+    }

tests/func/test_external_repo.py

@@ -205,6 +205,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
             repo.odb.local,
             PathInfo(repo.root_dir) / "dir",
             repo.repo_fs,
+            "md5",
             follow_subrepos=False,
         )
         save(repo.odb.local, obj)

tests/func/test_fs.py

@@ -349,3 +349,23 @@ def test_fs_ls_with_etag(dvc, cloud):
             fs.info(path_info.replace(path=details["name"]))["etag"]
             == details["etag"]
         )
+
+
+@pytest.mark.parametrize(
+    "cloud", [pytest.lazy_fixture("azure"), pytest.lazy_fixture("gs")]
+)
+def test_fs_fsspec_path_management(dvc, cloud):
+    cloud.gen({"foo": "foo", "data": {"bar": "bar", "baz": {"foo": "foo"}}})
+    fs = get_cloud_fs(dvc, **cloud.config)
+
+    root = cloud.parents[len(cloud.parents) - 1]
+    bucket_details = fs.info(root)
+
+    # special conditions: name always points to the bucket name
+    assert bucket_details["name"] == root.bucket
+    assert bucket_details["type"] == "directory"
+
+    data = cloud / "data"
+    data_details = fs.info(data)
+    assert data_details["name"].rstrip("/") == data.path
+    assert data_details["type"] == "directory"

tests/func/test_import_url.py

@@ -353,3 +353,42 @@ def test_import_url_to_remote_absolute(
 def test_import_url_to_remote_invalid_combinations(dvc):
     with pytest.raises(InvalidArgumentError, match="--no-exec"):
         dvc.imp_url("s3://bucket/foo", no_exec=True, to_remote=True)
+
+
+empty_xfail = pytest.mark.xfail(
+    reason="https://github.com/iterative/dvc/issues/5521"
+)
+
+
+@pytest.mark.parametrize(
+    "workspace",
+    [
+        pytest.lazy_fixture("s3"),
+        pytest.lazy_fixture("hdfs"),
+        pytest.param(pytest.lazy_fixture("gs"), marks=empty_xfail),
+        pytest.param(pytest.lazy_fixture("azure"), marks=empty_xfail),
+        pytest.param(
+            pytest.lazy_fixture("ssh"),
+            marks=pytest.mark.skipif(
+                os.name == "nt", reason="disabled on windows"
+            ),
+        ),
+    ],
+    indirect=True,
+)
+def test_import_url_empty_directory(tmp_dir, dvc, workspace):
+    # prefix based storage services (e.g s3) doesn't have the real concept
+    # of directories. So instead we create an empty file that ends with a
+    # trailing slash in order to actually support this operation
+    if workspace.IS_OBJECT_STORAGE:
+        contents = ""
+    else:
+        contents = {}
+
+    workspace.gen({"empty_dir/": contents})
+
+    dvc.imp_url("remote://workspace/empty_dir/")
+
+    empty_dir = tmp_dir / "empty_dir"
+    assert empty_dir.is_dir()
+    assert tuple(empty_dir.iterdir()) == ()

tests/func/test_odb.py

@@ -7,8 +7,8 @@ import pytest
 
 from dvc.hash_info import HashInfo
 from dvc.main import main
-from dvc.objects import ObjectFormatError
 from dvc.objects.db import ODBManager
+from dvc.objects.errors import ObjectFormatError
 from dvc.utils import relpath
 from tests.basic_env import TestDir, TestDvc
 

tests/func/test_remote.py

@@ -7,7 +7,6 @@ import pytest
 from mock import patch
 
 from dvc.config import Config
-from dvc.dir_info import DirInfo
 from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
 from dvc.fs.local import LocalFileSystem
 from dvc.main import main
@@ -147,26 +146,31 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
 
 def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
     from dvc.objects.stage import stage
+    from dvc.objects.tree import Tree
 
     tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
 
     path_info = PathInfo("data")
 
-    dir_info = DirInfo.from_list(
+    tree = Tree.from_list(
         [{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}]
     )
     with patch(
-        "dvc.objects.stage._collect_dir", return_value=dir_info,
+        "dvc.objects.stage._get_tree_obj", return_value=tree,
     ):
-        hash1 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info
+        hash1 = stage(
+            dvc.odb.local, path_info, dvc.odb.local.fs, "md5"
+        ).hash_info
 
-    dir_info = DirInfo.from_list(
+    tree = Tree.from_list(
         [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}]
     )
     with patch(
-        "dvc.objects.stage._collect_dir", return_value=dir_info,
+        "dvc.objects.stage._get_tree_obj", return_value=tree,
     ):
-        hash2 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info
+        hash2 = stage(
+            dvc.odb.local, path_info, dvc.odb.local.fs, "md5"
+        ).hash_info
 
     assert hash1 == hash2
 
@@ -278,7 +282,7 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
     remote = dvc.cloud.get_remote("upstream")
     foo_path = remote.odb.hash_to_path_info(foo.hash_info.value)
     bar_path = remote.odb.hash_to_path_info(
-        foo.hash_info.dir_info.trie[("bar",)].value
+        foo.obj.trie[("bar",)].hash_info.value
     )
     paths = [args[1] for args, _ in mocked_upload.call_args_list]
     assert paths.index(foo_path) > paths.index(bar_path)

tests/func/test_scm.py

@@ -1,6 +1,7 @@
 import os
 
 import pytest
+from flaky.flaky_decorator import flaky
 from git import Repo
 
 from dvc.scm import SCM, Git, NoSCM
@@ -242,6 +243,9 @@ def test_git_stash_drop(tmp_dir, scm, ref):
     assert len(stash) == 1
 
 
+# libgit2 stash_save() is flaky on linux when run inside pytest, see:
+# https://github.com/iterative/dvc/pull/5286#issuecomment-792574294
+@flaky(max_runs=5, min_passes=1)
 @pytest.mark.parametrize("ref", [None, "refs/foo/stash"])
 def test_git_stash_pop(tmp_dir, scm, ref):
     from dvc.scm.git import Stash
@@ -279,6 +283,7 @@ def test_git_stash_clear(tmp_dir, scm, ref):
 
     parts = list(stash.ref.split("/"))
     assert not os.path.exists(os.path.join(os.fspath(tmp_dir), ".git", *parts))
-    assert not os.path.exists(
-        os.path.join(os.fspath(tmp_dir), ".git", "logs", *parts)
-    )
+    # NOTE: some backends will completely remove reflog file on clear, some
+    # will only truncate it, either case means an empty stash
+    log_path = os.path.join(os.fspath(tmp_dir), ".git", "logs", *parts)
+    assert not os.path.exists(log_path) or not open(log_path).read()

tests/remotes/azure.py

@@ -22,6 +22,7 @@ TEST_AZURE_CONNECTION_STRING = (
 
 class Azure(Base, CloudURLInfo):
 
+    IS_OBJECT_STORAGE = True
     CONNECTION_STRING = None
 
     @cached_property

tests/remotes/base.py

@@ -7,6 +7,9 @@ from dvc.path_info import URLInfo
 
 
 class Base(URLInfo):
+
+    IS_OBJECT_STORAGE = False
+
     def is_file(self):
         raise NotImplementedError
 

tests/remotes/gs.py

@@ -23,6 +23,9 @@ os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
 
 
 class GCP(Base, CloudURLInfo):
+
+    IS_OBJECT_STORAGE = True
+
     @staticmethod
     def should_test():
         do_test = env2bool("DVC_TEST_GCP", undefined=None)