dvc: fix installation from archives

Fixes: #7124

.git_archival.txt

@@ -0,0 +1 @@
+ref-names: $Format:%D$

.gitattributes

@@ -0,0 +1 @@
+.git_archival.txt  export-subst

dvc/fs/azure.py

@@ -41,7 +41,6 @@ def _az_config():
 class AzureFileSystem(CallbackMixin, ObjectFSWrapper):
     scheme = Schemes.AZURE
     PARAM_CHECKSUM = "etag"
-    DETAIL_FIELDS = frozenset(("etag", "size"))
     REQUIRES = {
         "adlfs": "adlfs",
         "knack": "knack",

dvc/fs/base.py

@@ -4,7 +4,7 @@ import os
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from functools import partialmethod
 from multiprocessing import cpu_count
-from typing import ClassVar, Dict, FrozenSet, Optional
+from typing import ClassVar, Dict, Optional
 
 from funcy import cached_property
 from tqdm.utils import CallbackIOWrapper
@@ -43,7 +43,6 @@ class FileSystem:
     REQUIRES: ClassVar[Dict[str, str]] = {}
     _JOBS = 4 * cpu_count()
 
-    CHECKSUM_DIR_SUFFIX = ".dir"
     HASH_JOBS = max(1, min(4, cpu_count() // 2))
     LIST_OBJECT_PAGE_SIZE = 1000
     TRAVERSE_WEIGHT_MULTIPLIER = 5
@@ -55,7 +54,6 @@ class FileSystem:
     CHUNK_SIZE = 64 * 1024 * 1024  # 64 MiB
 
     PARAM_CHECKSUM: ClassVar[Optional[str]] = None
-    DETAIL_FIELDS: FrozenSet[str] = frozenset()
 
     def __init__(self, **kwargs):
         self._check_requires(**kwargs)
@@ -218,12 +216,6 @@ class FileSystem:
 
     # pylint: enable=unused-argument
 
-    @classmethod
-    def is_dir_hash(cls, hash_):
-        if not hash_:
-            return False
-        return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
-
     def upload(
         self,
         from_info,

dvc/fs/gs.py

@@ -12,7 +12,6 @@ class GSFileSystem(CallbackMixin, ObjectFSWrapper):
     scheme = Schemes.GS
     REQUIRES = {"gcsfs": "gcsfs"}
     PARAM_CHECKSUM = "etag"
-    DETAIL_FIELDS = frozenset(("etag", "size"))
 
     def _prepare_credentials(self, **config):
         login_info = {"consistency": None}

dvc/fs/oss.py

@@ -18,7 +18,6 @@ class OSSFileSystem(ObjectFSWrapper):
     PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
     LIST_OBJECT_PAGE_SIZE = 100
-    DETAIL_FIELDS = frozenset(("etag", "size"))
 
     def _prepare_credentials(self, **config):
         login_info = {}

dvc/fs/s3.py

@@ -18,7 +18,6 @@ class BaseS3FileSystem(ObjectFSWrapper):
     scheme = Schemes.S3
     REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"}
     PARAM_CHECKSUM = "etag"
-    DETAIL_FIELDS = frozenset(("etag", "size"))
 
     _GRANTS = {
         "grant_full_control": "GrantFullControl",

dvc/fs/webdav.py

@@ -17,7 +17,6 @@ class WebDAVFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
     TRAVERSE_PREFIX_LEN = 2
     REQUIRES = {"webdav4": "webdav4"}
     PARAM_CHECKSUM = "etag"
-    DETAIL_FIELDS = frozenset(("etag", "size"))
 
     def __init__(self, **config):
         super().__init__(**config)

dvc/objects/db/__init__.py

@@ -46,7 +46,6 @@ def get_index(odb) -> "ObjectDBIndexBase":
         hashlib.sha256(
             odb.fs.unstrip_protocol(odb.fs_path).encode("utf-8")
         ).hexdigest(),
-        odb.fs.CHECKSUM_DIR_SUFFIX,
     )
 
 

dvc/objects/db/base.py

@@ -372,17 +372,22 @@ class ObjectDB:
                     entry_obj.hash_info.value for _, entry_obj in tree
                 )
 
+        def _is_dir_hash(_hash):
+            from dvc.hash_info import HASH_DIR_SUFFIX
+
+            return _hash.endswith(HASH_DIR_SUFFIX)
+
         removed = False
         # hashes must be sorted to ensure we always remove .dir files first
         for hash_ in sorted(
             self.all(jobs, self.fs_path),
-            key=self.fs.is_dir_hash,
+            key=_is_dir_hash,
             reverse=True,
         ):
             if hash_ in used_hashes:
                 continue
             fs_path = self.hash_to_path(hash_)
-            if self.fs.is_dir_hash(hash_):
+            if _is_dir_hash(hash_):
                 # backward compatibility
                 # pylint: disable=protected-access
                 self._remove_unpacked_dir(hash_)

dvc/objects/db/index.py

@@ -1,7 +1,7 @@
 import logging
 import os
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Iterable, Optional, Set
+from typing import TYPE_CHECKING, Iterable, Set
 
 from ..errors import ObjectDBError
 
@@ -14,7 +14,9 @@ logger = logging.getLogger(__name__)
 class ObjectDBIndexBase(ABC):
     @abstractmethod
     def __init__(
-        self, tmp_dir: "StrPath", name: str, dir_suffix: Optional[str] = None
+        self,
+        tmp_dir: "StrPath",
+        name: str,
     ):
         pass
 
@@ -50,7 +52,9 @@ class ObjectDBIndexNoop(ObjectDBIndexBase):
     """No-op class for ODBs which are not indexed."""
 
     def __init__(
-        self, tmp_dir: "StrPath", name: str, dir_suffix: Optional[str] = None
+        self,
+        tmp_dir: "StrPath",
+        name: str,
     ):  # pylint: disable=super-init-not-called
         pass
 
@@ -80,7 +84,9 @@ class ObjectDBIndex(ObjectDBIndexBase):
     INDEX_DIR = "index"
 
     def __init__(
-        self, tmp_dir: "StrPath", name: str, dir_suffix: Optional[str] = None
+        self,
+        tmp_dir: "StrPath",
+        name: str,
     ):  # pylint: disable=super-init-not-called
         from diskcache import Index
 
@@ -92,10 +98,6 @@ class ObjectDBIndex(ObjectDBIndexBase):
         self.fs = LocalFileSystem()
         self.index = Index(self.index_dir)
 
-        if not dir_suffix:
-            dir_suffix = self.fs.CHECKSUM_DIR_SUFFIX
-        self.dir_suffix = dir_suffix
-
     def __iter__(self):
         return iter(self.index)
 
@@ -106,9 +108,6 @@ class ObjectDBIndex(ObjectDBIndexBase):
         """Iterate over .dir hashes stored in the index."""
         yield from (hash_ for hash_, is_dir in self.index.items() if is_dir)
 
-    def is_dir_hash(self, hash_: str):
-        return hash_.endswith(self.dir_suffix)
-
     def clear(self):
         """Clear this index (to force re-indexing later)."""
         from diskcache import Timeout

setup.cfg

@@ -28,6 +28,8 @@ classifiers =
 setup_requires =
     setuptools>=48
     setuptools_scm[toml]>=6.3.1
+    setuptools_scm_git_archive==1.1
+
 python_requires = >=3.7
 zip_safe = False
 packages = find:
@@ -73,7 +75,7 @@ install_requires =
     aiohttp-retry>=2.4.5
     diskcache>=5.2.1
     jaraco.windows>=5.7.0; python_version < '3.8' and sys_platform == 'win32'
-    scmrepo==0.0.3
+    scmrepo==0.0.4
 
 [options.extras_require]
 # gssapi should not be included in all_remotes, because it doesn't have wheels
@@ -98,12 +100,12 @@ azure =
     azure-identity>=1.4.0
     knack
 gdrive = pydrive2[fsspec]>=1.9.4
-gs = gcsfs==2021.11.0
+gs = gcsfs>=2021.11.1
 hdfs =
     fsspec[arrow]
-oss = ossfs==2021.8.0
+oss = ossfs>=2021.8.0
 s3 =
-    s3fs==2021.11.0
+    s3fs>=2021.11.1
     aiobotocore[boto3]>1.0.1
 ssh = sshfs[bcrypt]>=2021.11.2
 ssh_gssapi = sshfs[gssapi]>=2021.11.2
@@ -136,7 +138,7 @@ tests =
     xmltodict==0.12.0
     google-compute-engine==2.8.13
     google-cloud-storage==1.43.0
-    dvclive[image]==0.4.5
+    dvclive[image]==0.4.6
     hdfs==2.6.0
     # required by collective.checkdocs
     Pygments==2.10.0

tests/unit/remote/test_base.py

@@ -135,11 +135,3 @@ def test_list_paths(dvc):
         walk_mock.assert_called_with(
             posixpath.join(path, "00", "0"), prefix=True
         )
-
-
-@pytest.mark.parametrize(
-    "hash_, result",
-    [(None, False), ("", False), ("3456.dir", True), ("3456", False)],
-)
-def test_is_dir_hash(hash_, result):
-    assert FileSystem.is_dir_hash(hash_) == result

tests/unit/remote/test_index.py

@@ -16,11 +16,6 @@ def test_init(dvc, index):
     assert str(index.index_dir) == os.path.join(dvc.tmp_dir, "index", "foo")
 
 
-def test_is_dir_hash(dvc, index):
-    assert index.is_dir_hash("foo.dir")
-    assert not index.is_dir_hash("foo")
-
-
 def test_roundtrip(dvc, index):
     expected_dir = {"1234.dir"}
     expected_file = {"5678"}