mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
ref: use fs.checksum instead of get_mtime_and_size (#6495)
This commit is contained in:
@@ -124,6 +124,9 @@ class BaseFileSystem:
|
||||
f"dependencies: {missing}. {hint}"
|
||||
)
|
||||
|
||||
def checksum(self, path_info) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
|
||||
raise RemoteActionNotImplemented("open", self.scheme)
|
||||
|
||||
|
||||
@@ -261,3 +261,10 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
fs._download( # pylint: disable=protected-access
|
||||
path, to_file, **kwargs
|
||||
)
|
||||
|
||||
def checksum(self, path_info):
|
||||
info = self.info(path_info)
|
||||
md5 = info.get("md5")
|
||||
if md5:
|
||||
return md5
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -85,6 +85,9 @@ class FSSpecWrapper(BaseFileSystem):
|
||||
): # pylint: disable=arguments-differ
|
||||
return self.fs.open(self._with_bucket(path_info), mode=mode)
|
||||
|
||||
def checksum(self, path_info):
|
||||
return self.fs.checksum(self._with_bucket(path_info))
|
||||
|
||||
def copy(self, from_info, to_info):
|
||||
self.makedirs(to_info.parent)
|
||||
self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
|
||||
|
||||
+4
-15
@@ -1,6 +1,5 @@
|
||||
import errno
|
||||
import os
|
||||
import stat
|
||||
|
||||
from dvc.utils import is_exec, relpath
|
||||
|
||||
@@ -96,30 +95,20 @@ class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
if not self.exists(path_info):
|
||||
return False
|
||||
|
||||
mode = self.stat(path_info).st_mode
|
||||
mode = self.info(path_info)["mode"]
|
||||
return is_exec(mode)
|
||||
|
||||
def info(self, path_info):
|
||||
key = self._get_key(path_info)
|
||||
try:
|
||||
st = self.trie.stat(key)
|
||||
return self.trie.info(key)
|
||||
except KeyError:
|
||||
raise FileNotFoundError(
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), path_info
|
||||
)
|
||||
return {
|
||||
"size": st.st_size,
|
||||
"type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
|
||||
}
|
||||
|
||||
def stat(self, path):
|
||||
key = self._get_key(path)
|
||||
try:
|
||||
return self.trie.stat(key)
|
||||
except KeyError:
|
||||
raise FileNotFoundError(
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), path
|
||||
)
|
||||
def checksum(self, path_info):
|
||||
return self.info(path_info)["sha"]
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
for root, _, files in self.walk(path_info, **kwargs):
|
||||
|
||||
+6
-12
@@ -1,6 +1,5 @@
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scheme import Schemes
|
||||
@@ -36,6 +35,9 @@ class LocalFileSystem(BaseFileSystem):
|
||||
assert isinstance(path_info, str) or path_info.scheme == "local"
|
||||
return self.fs.exists(path_info)
|
||||
|
||||
def checksum(self, path_info) -> str:
|
||||
return self.fs.checksum(path_info)
|
||||
|
||||
def isfile(self, path_info) -> bool:
|
||||
return os.path.isfile(path_info)
|
||||
|
||||
@@ -83,12 +85,9 @@ class LocalFileSystem(BaseFileSystem):
|
||||
makedirs(path_info, exist_ok=kwargs.pop("exist_ok", True))
|
||||
|
||||
def isexec(self, path_info):
|
||||
mode = self.stat(path_info).st_mode
|
||||
mode = self.info(path_info)["mode"]
|
||||
return is_exec(mode)
|
||||
|
||||
def stat(self, path):
|
||||
return os.stat(path)
|
||||
|
||||
def move(self, from_info, to_info):
|
||||
if (
|
||||
isinstance(from_info, PathInfo) and from_info.scheme != "local"
|
||||
@@ -154,13 +153,8 @@ class LocalFileSystem(BaseFileSystem):
|
||||
def reflink(self, from_info, to_info):
|
||||
System.reflink(from_info, to_info)
|
||||
|
||||
@staticmethod
|
||||
def info(path_info):
|
||||
st = os.stat(path_info)
|
||||
return {
|
||||
"size": st.st_size,
|
||||
"type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
|
||||
}
|
||||
def info(self, path_info):
|
||||
return self.fs.info(path_info)
|
||||
|
||||
def _upload(
|
||||
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
|
||||
|
||||
+17
-14
@@ -1,6 +1,5 @@
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
import threading
|
||||
from contextlib import suppress
|
||||
from itertools import takewhile
|
||||
@@ -256,8 +255,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
return False
|
||||
|
||||
try:
|
||||
st = fs.stat(path)
|
||||
return stat.S_ISDIR(st.st_mode)
|
||||
info = fs.info(path)
|
||||
return info["type"] == "directory"
|
||||
except (OSError, ValueError):
|
||||
# from CPython's os.path.isdir()
|
||||
pass
|
||||
@@ -287,8 +286,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
return False
|
||||
|
||||
try:
|
||||
st = fs.stat(path)
|
||||
return stat.S_ISREG(st.st_mode)
|
||||
info = fs.info(path)
|
||||
return info["type"] == "file"
|
||||
except (OSError, ValueError):
|
||||
# from CPython's os.path.isfile()
|
||||
pass
|
||||
@@ -313,10 +312,6 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
return dvc_fs.isexec(path_info)
|
||||
return fs.isexec(path_info)
|
||||
|
||||
def stat(self, path):
|
||||
fs, _ = self._get_fs_pair(path)
|
||||
return fs.stat(path)
|
||||
|
||||
def _dvc_walk(self, walk):
|
||||
try:
|
||||
root, dirs, files = next(walk)
|
||||
@@ -478,11 +473,11 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
with suppress(FileNotFoundError):
|
||||
dvc_meta = dvc_fs.metadata(path_info)
|
||||
|
||||
stat_result = None
|
||||
info_result = None
|
||||
with suppress(FileNotFoundError):
|
||||
stat_result = fs.stat(path_info)
|
||||
info_result = fs.info(path_info)
|
||||
|
||||
if not stat_result and not dvc_meta:
|
||||
if not info_result and not dvc_meta:
|
||||
raise FileNotFoundError
|
||||
|
||||
from ._metadata import Metadata
|
||||
@@ -492,13 +487,13 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
repo=self._get_repo(abspath) or self._main_repo,
|
||||
)
|
||||
|
||||
isdir = bool(stat_result) and stat.S_ISDIR(stat_result.st_mode)
|
||||
isdir = bool(info_result) and info_result["type"] == "directory"
|
||||
meta.isdir = meta.isdir or isdir
|
||||
|
||||
if not dvc_meta:
|
||||
from dvc.utils import is_exec
|
||||
|
||||
meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode)
|
||||
meta.is_exec = bool(info_result) and is_exec(info_result["mode"])
|
||||
return meta
|
||||
|
||||
def info(self, path_info):
|
||||
@@ -508,3 +503,11 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
|
||||
return fs.info(path_info)
|
||||
except FileNotFoundError:
|
||||
return dvc_fs.info(path_info)
|
||||
|
||||
def checksum(self, path_info):
|
||||
fs, dvc_fs = self._get_fs_pair(path_info)
|
||||
|
||||
try:
|
||||
return fs.checksum(path_info)
|
||||
except FileNotFoundError:
|
||||
return dvc_fs.checksum(path_info)
|
||||
|
||||
@@ -2,7 +2,7 @@ import errno
|
||||
import logging
|
||||
import os
|
||||
import pickle
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from .errors import ObjectFormatError
|
||||
from .file import HashFile
|
||||
@@ -20,8 +20,7 @@ logger = logging.getLogger(__name__)
|
||||
class ReferenceHashFile(HashFile):
|
||||
PARAM_PATH = "path"
|
||||
PARAM_HASH = "hash"
|
||||
PARAM_MTIME = "mtime"
|
||||
PARAM_SIZE = "size"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
PARAM_FS_CONFIG = "fs_config"
|
||||
|
||||
def __init__(
|
||||
@@ -29,14 +28,11 @@ class ReferenceHashFile(HashFile):
|
||||
path_info: "AnyPath",
|
||||
fs: "BaseFileSystem",
|
||||
hash_info: "HashInfo",
|
||||
mtime: Optional[str] = None,
|
||||
size: Optional[int] = None,
|
||||
checksum: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(path_info, fs, hash_info, **kwargs)
|
||||
cur_mtime, cur_size = self._get_mtime_and_size()
|
||||
self.mtime = cur_mtime if mtime is None else mtime
|
||||
self.hash_info.size = cur_size if size is None else size
|
||||
self.checksum = checksum or fs.checksum(path_info)
|
||||
|
||||
def __str__(self):
|
||||
return f"ref object {self.hash_info} -> {self.path_info}"
|
||||
@@ -47,21 +43,14 @@ class ReferenceHashFile(HashFile):
|
||||
raise FileNotFoundError(
|
||||
errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
|
||||
)
|
||||
if not (self.mtime, self.size) == self._get_mtime_and_size():
|
||||
if self.checksum != self._get_checksum():
|
||||
raise ObjectFormatError(f"{self} is changed")
|
||||
if check_hash:
|
||||
self._check_hash(odb)
|
||||
|
||||
def _get_mtime_and_size(self) -> Tuple[Optional[str], int]:
|
||||
from dvc.utils.fs import get_mtime_and_size
|
||||
|
||||
def _get_checksum(self) -> str:
|
||||
assert self.fs
|
||||
if hasattr(self.fs, "stat"):
|
||||
try:
|
||||
return get_mtime_and_size(self.path_info, self.fs)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return None, self.fs.getsize(self.path_info)
|
||||
return self.fs.checksum(self.path_info)
|
||||
|
||||
def to_bytes(self):
|
||||
from dvc.fs.repo import RepoFileSystem
|
||||
@@ -79,8 +68,7 @@ class ReferenceHashFile(HashFile):
|
||||
dict_ = {
|
||||
self.PARAM_PATH: path_info,
|
||||
self.PARAM_HASH: self.hash_info,
|
||||
self.PARAM_MTIME: self.mtime,
|
||||
self.PARAM_SIZE: self.size,
|
||||
self.PARAM_CHECKSUM: self.checksum,
|
||||
self.PARAM_FS_CONFIG: self.config_tuple(self.fs),
|
||||
}
|
||||
try:
|
||||
@@ -118,8 +106,7 @@ class ReferenceHashFile(HashFile):
|
||||
path_info,
|
||||
fs,
|
||||
hash_info,
|
||||
mtime=dict_.get(cls.PARAM_MTIME),
|
||||
size=dict_.get(cls.PARAM_SIZE),
|
||||
checksum=dict_.get(cls.PARAM_CHECKSUM),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -39,7 +39,7 @@ class DulwichObject(GitObject):
|
||||
self.repo = repo
|
||||
self._name = name
|
||||
self._mode = mode
|
||||
self.sha = sha
|
||||
self._sha = sha
|
||||
|
||||
def open(self, mode: str = "r", encoding: str = None):
|
||||
if not encoding:
|
||||
@@ -68,6 +68,14 @@ class DulwichObject(GitObject):
|
||||
self.repo, entry.path.decode(), entry.mode, entry.sha
|
||||
)
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return len(self.repo[self.sha].as_raw_string())
|
||||
|
||||
@property
|
||||
def sha(self) -> str:
|
||||
return self._sha
|
||||
|
||||
|
||||
class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
"""Dulwich Git backend."""
|
||||
|
||||
@@ -92,6 +92,14 @@ class GitPythonObject(GitObject):
|
||||
for obj in self.obj:
|
||||
yield GitPythonObject(obj)
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self.obj.size
|
||||
|
||||
@property
|
||||
def sha(self) -> str:
|
||||
return self.obj.hexsha
|
||||
|
||||
|
||||
class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
"""git-python Git backend."""
|
||||
|
||||
@@ -58,6 +58,14 @@ class Pygit2Object(GitObject):
|
||||
return stat.S_IFDIR
|
||||
return self.obj.filemode
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return len(self.obj.read_raw())
|
||||
|
||||
@property
|
||||
def sha(self) -> str:
|
||||
return self.obj.hex
|
||||
|
||||
def scandir(self) -> Iterable["Pygit2Object"]:
|
||||
for entry in self.obj: # noqa: B301
|
||||
yield Pygit2Object(entry)
|
||||
|
||||
+17
-3
@@ -1,4 +1,3 @@
|
||||
import os
|
||||
import stat
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
@@ -44,6 +43,16 @@ class GitObject(ABC):
|
||||
def issubmodule(self) -> bool:
|
||||
return S_ISGITLINK(self.mode)
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def size(self) -> int:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def sha(self) -> str:
|
||||
pass
|
||||
|
||||
|
||||
class GitTrie:
|
||||
def __init__(self, tree: GitObject, rev: str):
|
||||
@@ -116,9 +125,14 @@ class GitTrie:
|
||||
if not topdown:
|
||||
yield top, dirs, nondirs
|
||||
|
||||
def stat(self, key: tuple) -> os.stat_result:
|
||||
def info(self, key: tuple) -> dict:
|
||||
obj = self.trie[key]
|
||||
return os.stat_result((obj.mode, 0, 0, 0, 0, 0, 0, 0, 0, 0))
|
||||
return {
|
||||
"size": obj.size,
|
||||
"type": "directory" if stat.S_ISDIR(obj.mode) else "file",
|
||||
"sha": obj.sha,
|
||||
"mode": obj.mode,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
+6
-6
@@ -47,22 +47,22 @@ def get_mtime_and_size(path, fs, dvcignore=None):
|
||||
walk_iterator = fs.walk_files(path)
|
||||
for file_path in walk_iterator:
|
||||
try:
|
||||
stats = fs.stat(file_path)
|
||||
stats = fs.info(file_path)
|
||||
except OSError as exc:
|
||||
# NOTE: broken symlink case.
|
||||
if exc.errno != errno.ENOENT:
|
||||
raise
|
||||
continue
|
||||
size += stats.st_size
|
||||
files_mtimes[os.fspath(file_path)] = stats.st_mtime
|
||||
size += stats["size"]
|
||||
files_mtimes[os.fspath(file_path)] = stats["mtime"]
|
||||
|
||||
# We track file changes and moves, which cannot be detected with simply
|
||||
# max(mtime(f) for f in non_ignored_files)
|
||||
mtime = dict_md5(files_mtimes)
|
||||
else:
|
||||
base_stat = fs.stat(path)
|
||||
size = base_stat.st_size
|
||||
mtime = base_stat.st_mtime
|
||||
base_stat = fs.info(path)
|
||||
size = base_stat["size"]
|
||||
mtime = base_stat["mtime"]
|
||||
mtime = int(nanotime.timestamp(mtime))
|
||||
|
||||
return str(mtime), size
|
||||
|
||||
Reference in New Issue
Block a user