diff --git a/dvc/fs/base.py b/dvc/fs/base.py index 5d992992f..cfb5306a8 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -124,6 +124,9 @@ class BaseFileSystem: f"dependencies: {missing}. {hint}" ) + def checksum(self, path_info) -> str: + raise NotImplementedError + def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs): raise RemoteActionNotImplemented("open", self.scheme) diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index b92caf861..36738e034 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -261,3 +261,10 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method fs._download( # pylint: disable=protected-access path, to_file, **kwargs ) + + def checksum(self, path_info): + info = self.info(path_info) + md5 = info.get("md5") + if md5: + return md5 + raise NotImplementedError diff --git a/dvc/fs/fsspec_wrapper.py b/dvc/fs/fsspec_wrapper.py index 335ec6987..cef7c6e51 100644 --- a/dvc/fs/fsspec_wrapper.py +++ b/dvc/fs/fsspec_wrapper.py @@ -85,6 +85,9 @@ class FSSpecWrapper(BaseFileSystem): ): # pylint: disable=arguments-differ return self.fs.open(self._with_bucket(path_info), mode=mode) + def checksum(self, path_info): + return self.fs.checksum(self._with_bucket(path_info)) + def copy(self, from_info, to_info): self.makedirs(to_info.parent) self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info)) diff --git a/dvc/fs/git.py b/dvc/fs/git.py index 46c4127cf..81eebdeb0 100644 --- a/dvc/fs/git.py +++ b/dvc/fs/git.py @@ -1,6 +1,5 @@ import errno import os -import stat from dvc.utils import is_exec, relpath @@ -96,30 +95,20 @@ class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method if not self.exists(path_info): return False - mode = self.stat(path_info).st_mode + mode = self.info(path_info)["mode"] return is_exec(mode) def info(self, path_info): key = self._get_key(path_info) try: - st = self.trie.stat(key) + return self.trie.info(key) except KeyError: raise FileNotFoundError( errno.ENOENT, os.strerror(errno.ENOENT), path_info ) - return { - "size": st.st_size, - "type": "directory" if stat.S_ISDIR(st.st_mode) else "file", - } - def stat(self, path): - key = self._get_key(path) - try: - return self.trie.stat(key) - except KeyError: - raise FileNotFoundError( - errno.ENOENT, os.strerror(errno.ENOENT), path - ) + def checksum(self, path_info): + return self.info(path_info)["sha"] def walk_files(self, path_info, **kwargs): for root, _, files in self.walk(path_info, **kwargs): diff --git a/dvc/fs/local.py b/dvc/fs/local.py index 4c02b54d9..9e3c24c75 100644 --- a/dvc/fs/local.py +++ b/dvc/fs/local.py @@ -1,6 +1,5 @@ import logging import os -import stat from dvc.path_info import PathInfo from dvc.scheme import Schemes @@ -36,6 +35,9 @@ class LocalFileSystem(BaseFileSystem): assert isinstance(path_info, str) or path_info.scheme == "local" return self.fs.exists(path_info) + def checksum(self, path_info) -> str: + return self.fs.checksum(path_info) + def isfile(self, path_info) -> bool: return os.path.isfile(path_info) @@ -83,12 +85,9 @@ class LocalFileSystem(BaseFileSystem): makedirs(path_info, exist_ok=kwargs.pop("exist_ok", True)) def isexec(self, path_info): - mode = self.stat(path_info).st_mode + mode = self.info(path_info)["mode"] return is_exec(mode) - def stat(self, path): - return os.stat(path) - def move(self, from_info, to_info): if ( isinstance(from_info, PathInfo) and from_info.scheme != "local" @@ -154,13 +153,8 @@ class LocalFileSystem(BaseFileSystem): def reflink(self, from_info, to_info): System.reflink(from_info, to_info) - @staticmethod - def info(path_info): - st = os.stat(path_info) - return { - "size": st.st_size, - "type": "directory" if stat.S_ISDIR(st.st_mode) else "file", - } + def info(self, path_info): + return self.fs.info(path_info) def _upload( self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs diff --git a/dvc/fs/repo.py b/dvc/fs/repo.py index c1517abbe..4152cbae0 100644 --- a/dvc/fs/repo.py +++ b/dvc/fs/repo.py @@ -1,6 +1,5 @@ import logging import os -import stat import threading from contextlib import suppress from itertools import takewhile @@ -256,8 +255,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method return False try: - st = fs.stat(path) - return stat.S_ISDIR(st.st_mode) + info = fs.info(path) + return info["type"] == "directory" except (OSError, ValueError): # from CPython's os.path.isdir() pass @@ -287,8 +286,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method return False try: - st = fs.stat(path) - return stat.S_ISREG(st.st_mode) + info = fs.info(path) + return info["type"] == "file" except (OSError, ValueError): # from CPython's os.path.isfile() pass @@ -313,10 +312,6 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method return dvc_fs.isexec(path_info) return fs.isexec(path_info) - def stat(self, path): - fs, _ = self._get_fs_pair(path) - return fs.stat(path) - def _dvc_walk(self, walk): try: root, dirs, files = next(walk) @@ -478,11 +473,11 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method with suppress(FileNotFoundError): dvc_meta = dvc_fs.metadata(path_info) - stat_result = None + info_result = None with suppress(FileNotFoundError): - stat_result = fs.stat(path_info) + info_result = fs.info(path_info) - if not stat_result and not dvc_meta: + if not info_result and not dvc_meta: raise FileNotFoundError from ._metadata import Metadata @@ -492,13 +487,13 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method repo=self._get_repo(abspath) or self._main_repo, ) - isdir = bool(stat_result) and stat.S_ISDIR(stat_result.st_mode) + isdir = bool(info_result) and info_result["type"] == "directory" meta.isdir = meta.isdir or isdir if not dvc_meta: from dvc.utils import is_exec - meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode) + meta.is_exec = bool(info_result) and is_exec(info_result["mode"]) return meta def info(self, path_info): @@ -508,3 +503,11 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method return fs.info(path_info) except FileNotFoundError: return dvc_fs.info(path_info) + + def checksum(self, path_info): + fs, dvc_fs = self._get_fs_pair(path_info) + + try: + return fs.checksum(path_info) + except FileNotFoundError: + return dvc_fs.checksum(path_info) diff --git a/dvc/objects/reference.py b/dvc/objects/reference.py index df6807917..1ba39ed8c 100644 --- a/dvc/objects/reference.py +++ b/dvc/objects/reference.py @@ -2,7 +2,7 @@ import errno import logging import os import pickle -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from .errors import ObjectFormatError from .file import HashFile @@ -20,8 +20,7 @@ logger = logging.getLogger(__name__) class ReferenceHashFile(HashFile): PARAM_PATH = "path" PARAM_HASH = "hash" - PARAM_MTIME = "mtime" - PARAM_SIZE = "size" + PARAM_CHECKSUM = "checksum" PARAM_FS_CONFIG = "fs_config" def __init__( @@ -29,14 +28,11 @@ class ReferenceHashFile(HashFile): path_info: "AnyPath", fs: "BaseFileSystem", hash_info: "HashInfo", - mtime: Optional[str] = None, - size: Optional[int] = None, + checksum: Optional[str] = None, **kwargs, ): super().__init__(path_info, fs, hash_info, **kwargs) - cur_mtime, cur_size = self._get_mtime_and_size() - self.mtime = cur_mtime if mtime is None else mtime - self.hash_info.size = cur_size if size is None else size + self.checksum = checksum or fs.checksum(path_info) def __str__(self): return f"ref object {self.hash_info} -> {self.path_info}" @@ -47,21 +43,14 @@ class ReferenceHashFile(HashFile): raise FileNotFoundError( errno.ENOENT, os.strerror(errno.ENOENT), self.path_info ) - if not (self.mtime, self.size) == self._get_mtime_and_size(): + if self.checksum != self._get_checksum(): raise ObjectFormatError(f"{self} is changed") if check_hash: self._check_hash(odb) - def _get_mtime_and_size(self) -> Tuple[Optional[str], int]: - from dvc.utils.fs import get_mtime_and_size - + def _get_checksum(self) -> str: assert self.fs - if hasattr(self.fs, "stat"): - try: - return get_mtime_and_size(self.path_info, self.fs) - except FileNotFoundError: - pass - return None, self.fs.getsize(self.path_info) + return self.fs.checksum(self.path_info) def to_bytes(self): from dvc.fs.repo import RepoFileSystem @@ -79,8 +68,7 @@ class ReferenceHashFile(HashFile): dict_ = { self.PARAM_PATH: path_info, self.PARAM_HASH: self.hash_info, - self.PARAM_MTIME: self.mtime, - self.PARAM_SIZE: self.size, + self.PARAM_CHECKSUM: self.checksum, self.PARAM_FS_CONFIG: self.config_tuple(self.fs), } try: @@ -118,8 +106,7 @@ class ReferenceHashFile(HashFile): path_info, fs, hash_info, - mtime=dict_.get(cls.PARAM_MTIME), - size=dict_.get(cls.PARAM_SIZE), + checksum=dict_.get(cls.PARAM_CHECKSUM), ) @staticmethod diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index 12e169142..1fbea38aa 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -39,7 +39,7 @@ class DulwichObject(GitObject): self.repo = repo self._name = name self._mode = mode - self.sha = sha + self._sha = sha def open(self, mode: str = "r", encoding: str = None): if not encoding: @@ -68,6 +68,14 @@ class DulwichObject(GitObject): self.repo, entry.path.decode(), entry.mode, entry.sha ) + @property + def size(self) -> int: + return len(self.repo[self.sha].as_raw_string()) + + @property + def sha(self) -> str: + return self._sha + class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method """Dulwich Git backend.""" diff --git a/dvc/scm/git/backend/gitpython.py b/dvc/scm/git/backend/gitpython.py index 4c9b40896..c982d91d9 100644 --- a/dvc/scm/git/backend/gitpython.py +++ b/dvc/scm/git/backend/gitpython.py @@ -92,6 +92,14 @@ class GitPythonObject(GitObject): for obj in self.obj: yield GitPythonObject(obj) + @property + def size(self) -> int: + return self.obj.size + + @property + def sha(self) -> str: + return self.obj.hexsha + class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method """git-python Git backend.""" diff --git a/dvc/scm/git/backend/pygit2.py b/dvc/scm/git/backend/pygit2.py index 907e33c77..7971e4d4b 100644 --- a/dvc/scm/git/backend/pygit2.py +++ b/dvc/scm/git/backend/pygit2.py @@ -58,6 +58,14 @@ class Pygit2Object(GitObject): return stat.S_IFDIR return self.obj.filemode + @property + def size(self) -> int: + return len(self.obj.read_raw()) + + @property + def sha(self) -> str: + return self.obj.hex + def scandir(self) -> Iterable["Pygit2Object"]: for entry in self.obj: # noqa: B301 yield Pygit2Object(entry) diff --git a/dvc/scm/git/objects.py b/dvc/scm/git/objects.py index 3b8ad2a3b..0988e0f21 100644 --- a/dvc/scm/git/objects.py +++ b/dvc/scm/git/objects.py @@ -1,4 +1,3 @@ -import os import stat from abc import ABC, abstractmethod from dataclasses import dataclass @@ -44,6 +43,16 @@ class GitObject(ABC): def issubmodule(self) -> bool: return S_ISGITLINK(self.mode) + @property + @abstractmethod + def size(self) -> int: + pass + + @property + @abstractmethod + def sha(self) -> str: + pass + class GitTrie: def __init__(self, tree: GitObject, rev: str): @@ -116,9 +125,14 @@ class GitTrie: if not topdown: yield top, dirs, nondirs - def stat(self, key: tuple) -> os.stat_result: + def info(self, key: tuple) -> dict: obj = self.trie[key] - return os.stat_result((obj.mode, 0, 0, 0, 0, 0, 0, 0, 0, 0)) + return { + "size": obj.size, + "type": "directory" if stat.S_ISDIR(obj.mode) else "file", + "sha": obj.sha, + "mode": obj.mode, + } @dataclass diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index bfcf4cb5f..6c5d10b37 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -47,22 +47,22 @@ def get_mtime_and_size(path, fs, dvcignore=None): walk_iterator = fs.walk_files(path) for file_path in walk_iterator: try: - stats = fs.stat(file_path) + stats = fs.info(file_path) except OSError as exc: # NOTE: broken symlink case. if exc.errno != errno.ENOENT: raise continue - size += stats.st_size - files_mtimes[os.fspath(file_path)] = stats.st_mtime + size += stats["size"] + files_mtimes[os.fspath(file_path)] = stats["mtime"] # We track file changes and moves, which cannot be detected with simply # max(mtime(f) for f in non_ignored_files) mtime = dict_md5(files_mtimes) else: - base_stat = fs.stat(path) - size = base_stat.st_size - mtime = base_stat.st_mtime + base_stat = fs.info(path) + size = base_stat["size"] + mtime = base_stat["mtime"] mtime = int(nanotime.timestamp(mtime)) return str(mtime), size