ref: use fs.checksum instead of get_mtime_and_size (#6495)

This commit is contained in:
Ruslan Kuprieiev
2021-08-27 12:37:08 +09:00
committed by GitHub
parent ab08db5065
commit 95ba68d181
12 changed files with 97 additions and 73 deletions
+3
View File
@@ -124,6 +124,9 @@ class BaseFileSystem:
f"dependencies: {missing}. {hint}"
)
def checksum(self, path_info) -> str:
raise NotImplementedError
def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
raise RemoteActionNotImplemented("open", self.scheme)
+7
View File
@@ -261,3 +261,10 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
fs._download( # pylint: disable=protected-access
path, to_file, **kwargs
)
def checksum(self, path_info):
info = self.info(path_info)
md5 = info.get("md5")
if md5:
return md5
raise NotImplementedError
+3
View File
@@ -85,6 +85,9 @@ class FSSpecWrapper(BaseFileSystem):
): # pylint: disable=arguments-differ
return self.fs.open(self._with_bucket(path_info), mode=mode)
def checksum(self, path_info):
return self.fs.checksum(self._with_bucket(path_info))
def copy(self, from_info, to_info):
self.makedirs(to_info.parent)
self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
+4 -15
View File
@@ -1,6 +1,5 @@
import errno
import os
import stat
from dvc.utils import is_exec, relpath
@@ -96,30 +95,20 @@ class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method
if not self.exists(path_info):
return False
mode = self.stat(path_info).st_mode
mode = self.info(path_info)["mode"]
return is_exec(mode)
def info(self, path_info):
key = self._get_key(path_info)
try:
st = self.trie.stat(key)
return self.trie.info(key)
except KeyError:
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
return {
"size": st.st_size,
"type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
}
def stat(self, path):
key = self._get_key(path)
try:
return self.trie.stat(key)
except KeyError:
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path
)
def checksum(self, path_info):
return self.info(path_info)["sha"]
def walk_files(self, path_info, **kwargs):
for root, _, files in self.walk(path_info, **kwargs):
+6 -12
View File
@@ -1,6 +1,5 @@
import logging
import os
import stat
from dvc.path_info import PathInfo
from dvc.scheme import Schemes
@@ -36,6 +35,9 @@ class LocalFileSystem(BaseFileSystem):
assert isinstance(path_info, str) or path_info.scheme == "local"
return self.fs.exists(path_info)
def checksum(self, path_info) -> str:
return self.fs.checksum(path_info)
def isfile(self, path_info) -> bool:
return os.path.isfile(path_info)
@@ -83,12 +85,9 @@ class LocalFileSystem(BaseFileSystem):
makedirs(path_info, exist_ok=kwargs.pop("exist_ok", True))
def isexec(self, path_info):
mode = self.stat(path_info).st_mode
mode = self.info(path_info)["mode"]
return is_exec(mode)
def stat(self, path):
return os.stat(path)
def move(self, from_info, to_info):
if (
isinstance(from_info, PathInfo) and from_info.scheme != "local"
@@ -154,13 +153,8 @@ class LocalFileSystem(BaseFileSystem):
def reflink(self, from_info, to_info):
System.reflink(from_info, to_info)
@staticmethod
def info(path_info):
st = os.stat(path_info)
return {
"size": st.st_size,
"type": "directory" if stat.S_ISDIR(st.st_mode) else "file",
}
def info(self, path_info):
return self.fs.info(path_info)
def _upload(
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
+17 -14
View File
@@ -1,6 +1,5 @@
import logging
import os
import stat
import threading
from contextlib import suppress
from itertools import takewhile
@@ -256,8 +255,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
return False
try:
st = fs.stat(path)
return stat.S_ISDIR(st.st_mode)
info = fs.info(path)
return info["type"] == "directory"
except (OSError, ValueError):
# from CPython's os.path.isdir()
pass
@@ -287,8 +286,8 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
return False
try:
st = fs.stat(path)
return stat.S_ISREG(st.st_mode)
info = fs.info(path)
return info["type"] == "file"
except (OSError, ValueError):
# from CPython's os.path.isfile()
pass
@@ -313,10 +312,6 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
return dvc_fs.isexec(path_info)
return fs.isexec(path_info)
def stat(self, path):
fs, _ = self._get_fs_pair(path)
return fs.stat(path)
def _dvc_walk(self, walk):
try:
root, dirs, files = next(walk)
@@ -478,11 +473,11 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
with suppress(FileNotFoundError):
dvc_meta = dvc_fs.metadata(path_info)
stat_result = None
info_result = None
with suppress(FileNotFoundError):
stat_result = fs.stat(path_info)
info_result = fs.info(path_info)
if not stat_result and not dvc_meta:
if not info_result and not dvc_meta:
raise FileNotFoundError
from ._metadata import Metadata
@@ -492,13 +487,13 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
repo=self._get_repo(abspath) or self._main_repo,
)
isdir = bool(stat_result) and stat.S_ISDIR(stat_result.st_mode)
isdir = bool(info_result) and info_result["type"] == "directory"
meta.isdir = meta.isdir or isdir
if not dvc_meta:
from dvc.utils import is_exec
meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode)
meta.is_exec = bool(info_result) and is_exec(info_result["mode"])
return meta
def info(self, path_info):
@@ -508,3 +503,11 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method
return fs.info(path_info)
except FileNotFoundError:
return dvc_fs.info(path_info)
def checksum(self, path_info):
fs, dvc_fs = self._get_fs_pair(path_info)
try:
return fs.checksum(path_info)
except FileNotFoundError:
return dvc_fs.checksum(path_info)
+9 -22
View File
@@ -2,7 +2,7 @@ import errno
import logging
import os
import pickle
from typing import TYPE_CHECKING, Optional, Tuple
from typing import TYPE_CHECKING, Optional
from .errors import ObjectFormatError
from .file import HashFile
@@ -20,8 +20,7 @@ logger = logging.getLogger(__name__)
class ReferenceHashFile(HashFile):
PARAM_PATH = "path"
PARAM_HASH = "hash"
PARAM_MTIME = "mtime"
PARAM_SIZE = "size"
PARAM_CHECKSUM = "checksum"
PARAM_FS_CONFIG = "fs_config"
def __init__(
@@ -29,14 +28,11 @@ class ReferenceHashFile(HashFile):
path_info: "AnyPath",
fs: "BaseFileSystem",
hash_info: "HashInfo",
mtime: Optional[str] = None,
size: Optional[int] = None,
checksum: Optional[str] = None,
**kwargs,
):
super().__init__(path_info, fs, hash_info, **kwargs)
cur_mtime, cur_size = self._get_mtime_and_size()
self.mtime = cur_mtime if mtime is None else mtime
self.hash_info.size = cur_size if size is None else size
self.checksum = checksum or fs.checksum(path_info)
def __str__(self):
return f"ref object {self.hash_info} -> {self.path_info}"
@@ -47,21 +43,14 @@ class ReferenceHashFile(HashFile):
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
)
if not (self.mtime, self.size) == self._get_mtime_and_size():
if self.checksum != self._get_checksum():
raise ObjectFormatError(f"{self} is changed")
if check_hash:
self._check_hash(odb)
def _get_mtime_and_size(self) -> Tuple[Optional[str], int]:
from dvc.utils.fs import get_mtime_and_size
def _get_checksum(self) -> str:
assert self.fs
if hasattr(self.fs, "stat"):
try:
return get_mtime_and_size(self.path_info, self.fs)
except FileNotFoundError:
pass
return None, self.fs.getsize(self.path_info)
return self.fs.checksum(self.path_info)
def to_bytes(self):
from dvc.fs.repo import RepoFileSystem
@@ -79,8 +68,7 @@ class ReferenceHashFile(HashFile):
dict_ = {
self.PARAM_PATH: path_info,
self.PARAM_HASH: self.hash_info,
self.PARAM_MTIME: self.mtime,
self.PARAM_SIZE: self.size,
self.PARAM_CHECKSUM: self.checksum,
self.PARAM_FS_CONFIG: self.config_tuple(self.fs),
}
try:
@@ -118,8 +106,7 @@ class ReferenceHashFile(HashFile):
path_info,
fs,
hash_info,
mtime=dict_.get(cls.PARAM_MTIME),
size=dict_.get(cls.PARAM_SIZE),
checksum=dict_.get(cls.PARAM_CHECKSUM),
)
@staticmethod
+9 -1
View File
@@ -39,7 +39,7 @@ class DulwichObject(GitObject):
self.repo = repo
self._name = name
self._mode = mode
self.sha = sha
self._sha = sha
def open(self, mode: str = "r", encoding: str = None):
if not encoding:
@@ -68,6 +68,14 @@ class DulwichObject(GitObject):
self.repo, entry.path.decode(), entry.mode, entry.sha
)
@property
def size(self) -> int:
return len(self.repo[self.sha].as_raw_string())
@property
def sha(self) -> str:
return self._sha
class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
"""Dulwich Git backend."""
+8
View File
@@ -92,6 +92,14 @@ class GitPythonObject(GitObject):
for obj in self.obj:
yield GitPythonObject(obj)
@property
def size(self) -> int:
return self.obj.size
@property
def sha(self) -> str:
return self.obj.hexsha
class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
"""git-python Git backend."""
+8
View File
@@ -58,6 +58,14 @@ class Pygit2Object(GitObject):
return stat.S_IFDIR
return self.obj.filemode
@property
def size(self) -> int:
return len(self.obj.read_raw())
@property
def sha(self) -> str:
return self.obj.hex
def scandir(self) -> Iterable["Pygit2Object"]:
for entry in self.obj: # noqa: B301
yield Pygit2Object(entry)
+17 -3
View File
@@ -1,4 +1,3 @@
import os
import stat
from abc import ABC, abstractmethod
from dataclasses import dataclass
@@ -44,6 +43,16 @@ class GitObject(ABC):
def issubmodule(self) -> bool:
return S_ISGITLINK(self.mode)
@property
@abstractmethod
def size(self) -> int:
pass
@property
@abstractmethod
def sha(self) -> str:
pass
class GitTrie:
def __init__(self, tree: GitObject, rev: str):
@@ -116,9 +125,14 @@ class GitTrie:
if not topdown:
yield top, dirs, nondirs
def stat(self, key: tuple) -> os.stat_result:
def info(self, key: tuple) -> dict:
obj = self.trie[key]
return os.stat_result((obj.mode, 0, 0, 0, 0, 0, 0, 0, 0, 0))
return {
"size": obj.size,
"type": "directory" if stat.S_ISDIR(obj.mode) else "file",
"sha": obj.sha,
"mode": obj.mode,
}
@dataclass
+6 -6
View File
@@ -47,22 +47,22 @@ def get_mtime_and_size(path, fs, dvcignore=None):
walk_iterator = fs.walk_files(path)
for file_path in walk_iterator:
try:
stats = fs.stat(file_path)
stats = fs.info(file_path)
except OSError as exc:
# NOTE: broken symlink case.
if exc.errno != errno.ENOENT:
raise
continue
size += stats.st_size
files_mtimes[os.fspath(file_path)] = stats.st_mtime
size += stats["size"]
files_mtimes[os.fspath(file_path)] = stats["mtime"]
# We track file changes and moves, which cannot be detected with simply
# max(mtime(f) for f in non_ignored_files)
mtime = dict_md5(files_mtimes)
else:
base_stat = fs.stat(path)
size = base_stat.st_size
mtime = base_stat.st_mtime
base_stat = fs.info(path)
size = base_stat["size"]
mtime = base_stat["mtime"]
mtime = int(nanotime.timestamp(mtime))
return str(mtime), size