From 11eaa87849cbe4522798857ed98f3c8e18f06f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Thu, 12 May 2022 17:51:34 +0545 Subject: [PATCH] add typings to istextfile and system Also moves umask to dvc.fs.system --- dvc/data/db/local.py | 3 ++- dvc/fs/system.py | 30 +++++++++++++++++++----------- dvc/objects/istextfile.py | 9 +++++++-- dvc/utils/fs.py | 3 --- tests/func/test_odb.py | 2 +- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/dvc/data/db/local.py b/dvc/data/db/local.py index e21e5b958..ceae2439e 100644 --- a/dvc/data/db/local.py +++ b/dvc/data/db/local.py @@ -6,11 +6,12 @@ from functools import partial from funcy import cached_property from shortuuid import uuid +from dvc.fs.system import umask from dvc.objects.db import ObjectDB, noop, wrap_iter from dvc.objects.errors import ObjectFormatError from dvc.objects.hash_info import HashInfo from dvc.utils import relpath -from dvc.utils.fs import copyfile, remove, umask, walk_files +from dvc.utils.fs import copyfile, remove, walk_files logger = logging.getLogger(__name__) diff --git a/dvc/fs/system.py b/dvc/fs/system.py index af1137a24..1c9d54b12 100644 --- a/dvc/fs/system.py +++ b/dvc/fs/system.py @@ -4,6 +4,10 @@ import os import platform import stat import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .base import AnyFSPath logger = logging.getLogger(__name__) @@ -21,7 +25,11 @@ else: realpath = os.path.realpath -def hardlink(source, link_name): +umask = os.umask(0) +os.umask(umask) + + +def hardlink(source: "AnyFSPath", link_name: "AnyFSPath") -> None: # NOTE: we should really be using `os.link()` here with # `follow_symlinks=True`, but unfortunately the implementation is # buggy across platforms, so until it is fixed, we just dereference @@ -37,11 +45,11 @@ def hardlink(source, link_name): os.link(src, link_name) -def symlink(source, link_name): +def symlink(source: "AnyFSPath", link_name: "AnyFSPath") -> None: os.symlink(source, link_name) -def _reflink_darwin(src, dst): +def _reflink_darwin(src: "AnyFSPath", dst: "AnyFSPath") -> None: import ctypes def _cdll(name): @@ -82,7 +90,7 @@ def _reflink_darwin(src, dst): raise OSError(err, msg) -def _reflink_linux(src, dst): +def _reflink_linux(src: "AnyFSPath", dst: "AnyFSPath") -> None: import fcntl # pylint: disable=import-error from contextlib import suppress @@ -97,9 +105,7 @@ def _reflink_linux(src, dst): raise -def reflink(source, link_name): - from dvc.utils.fs import umask - +def reflink(source: "AnyFSPath", link_name: "AnyFSPath") -> None: source, link_name = os.fspath(source), os.fspath(link_name) system = platform.system() @@ -118,15 +124,17 @@ def reflink(source, link_name): os.chmod(link_name, 0o666 & ~umask) -def inode(path): +def inode(path: "AnyFSPath") -> int: ino = os.lstat(path).st_ino - logger.trace("Path '%s' inode '%d'", path, ino) + logger.trace( # type: ignore[attr-defined] + "Path '%s' inode '%d'", path, ino + ) return ino -def is_symlink(path): +def is_symlink(path: "AnyFSPath") -> bool: return os.path.islink(path) -def is_hardlink(path): +def is_hardlink(path: "AnyFSPath") -> bool: return os.stat(path).st_nlink > 1 diff --git a/dvc/objects/istextfile.py b/dvc/objects/istextfile.py index 81144927c..6501bb694 100644 --- a/dvc/objects/istextfile.py +++ b/dvc/objects/istextfile.py @@ -2,13 +2,16 @@ # Based on https://eli.thegreenplace.net/2011/10/19/ # perls-guess-if-file-is-text-or-binary-implemented-in-python +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from dvc.fs.base import AnyFSPath, FileSystem TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b" DEFAULT_CHUNK_SIZE = 512 -def istextblock(block): +def istextblock(block: bytes) -> bool: if not block: # An empty file is considered a valid text file return True @@ -23,7 +26,9 @@ def istextblock(block): return float(len(nontext)) / len(block) <= 0.30 -def istextfile(fname, fs, blocksize=DEFAULT_CHUNK_SIZE): +def istextfile( + fname: "AnyFSPath", fs: "FileSystem", blocksize: int = DEFAULT_CHUNK_SIZE +) -> bool: """Uses heuristics to guess whether the given file is text or binary, by reading a single block of bytes from the file. If more than 30% of the chars in the block are non-text, or there diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index 6cec4603a..320ffadb0 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -18,9 +18,6 @@ logger = logging.getLogger(__name__) LOCAL_CHUNK_SIZE = 2**20 # 1 MB -umask = os.umask(0) -os.umask(umask) - def get_mtime_and_size(path, fs, dvcignore=None): import nanotime diff --git a/tests/func/test_odb.py b/tests/func/test_odb.py index e35319efa..650f38398 100644 --- a/tests/func/test_odb.py +++ b/tests/func/test_odb.py @@ -180,7 +180,7 @@ def test_default_cache_type(dvc): @pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") @pytest.mark.parametrize("group", [False, True]) def test_shared_cache(tmp_dir, dvc, group): - from dvc.utils.fs import umask + from dvc.fs.system import umask if group: with dvc.config.edit() as conf: