add typings to istextfile and system

Also moves umask to dvc.fs.system
This commit is contained in:
Saugat Pachhai (सौगात)
2022-05-12 18:02:42 +03:00
committed by Ruslan Kuprieiev
parent b4faa5616f
commit 11eaa87849
5 changed files with 29 additions and 18 deletions
+2 -1
View File
@@ -6,11 +6,12 @@ from functools import partial
from funcy import cached_property
from shortuuid import uuid
from dvc.fs.system import umask
from dvc.objects.db import ObjectDB, noop, wrap_iter
from dvc.objects.errors import ObjectFormatError
from dvc.objects.hash_info import HashInfo
from dvc.utils import relpath
from dvc.utils.fs import copyfile, remove, umask, walk_files
from dvc.utils.fs import copyfile, remove, walk_files
logger = logging.getLogger(__name__)
+19 -11
View File
@@ -4,6 +4,10 @@ import os
import platform
import stat
import sys
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .base import AnyFSPath
logger = logging.getLogger(__name__)
@@ -21,7 +25,11 @@ else:
realpath = os.path.realpath
def hardlink(source, link_name):
umask = os.umask(0)
os.umask(umask)
def hardlink(source: "AnyFSPath", link_name: "AnyFSPath") -> None:
# NOTE: we should really be using `os.link()` here with
# `follow_symlinks=True`, but unfortunately the implementation is
# buggy across platforms, so until it is fixed, we just dereference
@@ -37,11 +45,11 @@ def hardlink(source, link_name):
os.link(src, link_name)
def symlink(source, link_name):
def symlink(source: "AnyFSPath", link_name: "AnyFSPath") -> None:
os.symlink(source, link_name)
def _reflink_darwin(src, dst):
def _reflink_darwin(src: "AnyFSPath", dst: "AnyFSPath") -> None:
import ctypes
def _cdll(name):
@@ -82,7 +90,7 @@ def _reflink_darwin(src, dst):
raise OSError(err, msg)
def _reflink_linux(src, dst):
def _reflink_linux(src: "AnyFSPath", dst: "AnyFSPath") -> None:
import fcntl # pylint: disable=import-error
from contextlib import suppress
@@ -97,9 +105,7 @@ def _reflink_linux(src, dst):
raise
def reflink(source, link_name):
from dvc.utils.fs import umask
def reflink(source: "AnyFSPath", link_name: "AnyFSPath") -> None:
source, link_name = os.fspath(source), os.fspath(link_name)
system = platform.system()
@@ -118,15 +124,17 @@ def reflink(source, link_name):
os.chmod(link_name, 0o666 & ~umask)
def inode(path):
def inode(path: "AnyFSPath") -> int:
ino = os.lstat(path).st_ino
logger.trace("Path '%s' inode '%d'", path, ino)
logger.trace( # type: ignore[attr-defined]
"Path '%s' inode '%d'", path, ino
)
return ino
def is_symlink(path):
def is_symlink(path: "AnyFSPath") -> bool:
return os.path.islink(path)
def is_hardlink(path):
def is_hardlink(path: "AnyFSPath") -> bool:
return os.stat(path).st_nlink > 1
+7 -2
View File
@@ -2,13 +2,16 @@
# Based on https://eli.thegreenplace.net/2011/10/19/
# perls-guess-if-file-is-text-or-binary-implemented-in-python
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from dvc.fs.base import AnyFSPath, FileSystem
TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b"
DEFAULT_CHUNK_SIZE = 512
def istextblock(block):
def istextblock(block: bytes) -> bool:
if not block:
# An empty file is considered a valid text file
return True
@@ -23,7 +26,9 @@ def istextblock(block):
return float(len(nontext)) / len(block) <= 0.30
def istextfile(fname, fs, blocksize=DEFAULT_CHUNK_SIZE):
def istextfile(
fname: "AnyFSPath", fs: "FileSystem", blocksize: int = DEFAULT_CHUNK_SIZE
) -> bool:
"""Uses heuristics to guess whether the given file is text or binary,
by reading a single block of bytes from the file.
If more than 30% of the chars in the block are non-text, or there
-3
View File
@@ -18,9 +18,6 @@ logger = logging.getLogger(__name__)
LOCAL_CHUNK_SIZE = 2**20 # 1 MB
umask = os.umask(0)
os.umask(umask)
def get_mtime_and_size(path, fs, dvcignore=None):
import nanotime
+1 -1
View File
@@ -180,7 +180,7 @@ def test_default_cache_type(dvc):
@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
@pytest.mark.parametrize("group", [False, True])
def test_shared_cache(tmp_dir, dvc, group):
from dvc.utils.fs import umask
from dvc.fs.system import umask
if group:
with dvc.config.edit() as conf: