diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 72bde1b62..a6c79f04d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,11 +69,13 @@ repos: files: ^dvc/ language: system types: [python] + require_serial: true - id: pylint name: pylint entry: pylint language: system types: [python] + require_serial: true - hooks: - args: - git-hook diff --git a/dvc/_debug.py b/dvc/_debug.py index 94cc5b74f..4197619b2 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -29,7 +29,7 @@ def viztracer_profile(path: Union[Callable[[], str], str], depth: int = -1): @contextmanager def yappi_profile( - path: Union[Callable[[], str], str] = None, + path: Optional[Union[Callable[[], str], str]] = None, wall_clock: Optional[bool] = True, separate_threads: Optional[bool] = False, ): @@ -95,7 +95,7 @@ def instrument(html_output=False): @contextmanager -def profile(dump_path: str = None): +def profile(dump_path: Optional[str] = None): """Run a cprofile""" import cProfile @@ -158,7 +158,7 @@ def _get_path_func(tool: str, ext: str): @contextmanager -def debugtools(args: "Namespace" = None, **kwargs): +def debugtools(args: Optional["Namespace"] = None, **kwargs): kw = vars(args) if args else {} kw.update(kwargs) diff --git a/dvc/compare.py b/dvc/compare.py index 6f6c77a81..bbd19a406 100644 --- a/dvc/compare.py +++ b/dvc/compare.py @@ -167,7 +167,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): return buff.getvalue() def to_parallel_coordinates( - self, output_path: "StrPath", color_by: str = None + self, output_path: "StrPath", color_by: Optional[str] = None ) -> "StrPath": from dvc_render.html import render_html from dvc_render.plotly import ParallelCoordinatesRenderer @@ -209,7 +209,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): ui.table(self, headers=self.keys(), **kwargs) def as_dict( - self, cols: Iterable[str] = None + self, cols: Optional[Iterable[str]] = None ) -> Iterable[Dict[str, "CellT"]]: keys = self.keys() if cols is None else set(cols) return [ @@ -324,7 +324,7 @@ def _normalize_float(val: float, precision: int): def _format_field( - val: Any, precision: int = None, round_digits: bool = False + val: Any, precision: Optional[int] = None, round_digits: bool = False ) -> str: def _format(_val): if isinstance(_val, float) and precision: @@ -345,11 +345,11 @@ def diff_table( old: bool = True, no_path: bool = False, show_changes: bool = True, - precision: int = None, + precision: Optional[int] = None, round_digits: bool = False, - on_empty_diff: str = None, - a_rev: str = None, - b_rev: str = None, + on_empty_diff: Optional[str] = None, + a_rev: Optional[str] = None, + b_rev: Optional[str] = None, ) -> TabularData: a_rev = a_rev or "HEAD" b_rev = b_rev or "workspace" @@ -393,12 +393,12 @@ def show_diff( old: bool = True, no_path: bool = False, show_changes: bool = True, - precision: int = None, + precision: Optional[int] = None, round_digits: bool = False, - on_empty_diff: str = None, + on_empty_diff: Optional[str] = None, markdown: bool = False, - a_rev: str = None, - b_rev: str = None, + a_rev: Optional[str] = None, + b_rev: Optional[str] = None, ) -> None: td = diff_table( diff, @@ -420,7 +420,7 @@ def metrics_table( all_branches: bool = False, all_tags: bool = False, all_commits: bool = False, - precision: int = None, + precision: Optional[int] = None, round_digits: bool = False, ): from dvc.utils.diff import format_dict @@ -460,7 +460,7 @@ def show_metrics( all_branches: bool = False, all_tags: bool = False, all_commits: bool = False, - precision: int = None, + precision: Optional[int] = None, round_digits: bool = False, ) -> None: td = metrics_table( diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index e71fed463..7cfd89259 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -11,8 +11,8 @@ from dvc_data.hashfile.meta import Meta from .base import Dependency if TYPE_CHECKING: - from dvc_data.hashfile.file import HashFile from dvc_data.hashfile.hash_info import HashInfo + from dvc_data.hashfile.obj import HashFile from dvc_objects.db import ObjectDB diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index babee1431..a1184ab5f 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -7,6 +7,7 @@ from typing import ( Callable, Dict, List, + Optional, Tuple, TypeVar, Union, @@ -162,7 +163,7 @@ class FileMixin: return self._load_yaml(**kwargs) @classmethod - def validate(cls, d: _T, fname: str = None) -> _T: + def validate(cls, d: _T, fname: Optional[str] = None) -> _T: from dvc.utils.strictyaml import validate return validate(d, cls.SCHEMA, path=fname) # type: ignore[arg-type] diff --git a/dvc/fs/callbacks.py b/dvc/fs/callbacks.py index 48d6db66e..68a8c5cdb 100644 --- a/dvc/fs/callbacks.py +++ b/dvc/fs/callbacks.py @@ -20,10 +20,10 @@ class RichCallback(Callback): self, size: Optional[int] = None, value: int = 0, - progress: "RichTransferProgress" = None, - desc: str = None, + progress: Optional["RichTransferProgress"] = None, + desc: Optional[str] = None, bytes: bool = False, # noqa: A002, pylint: disable=redefined-builtin - unit: str = None, + unit: Optional[str] = None, disable: bool = False, transient: bool = True, ) -> None: @@ -83,7 +83,7 @@ class RichCallback(Callback): path_1: Union[str, BinaryIO], path_2: str, kwargs: Dict[str, Any], - child: "Callback" = None, + child: Optional["Callback"] = None, ): child = child or RichCallback( progress=self.progress, diff --git a/dvc/fs/git.py b/dvc/fs/git.py index 00ddebbf0..383e7529b 100644 --- a/dvc/fs/git.py +++ b/dvc/fs/git.py @@ -1,5 +1,5 @@ import functools -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional from . import FileSystem @@ -17,10 +17,10 @@ class GitFileSystem(FileSystem): # pylint:disable=abstract-method def __init__( self, - path: str = None, - rev: str = None, - scm: "Git" = None, - trie: "GitTrie" = None, + path: Optional[str] = None, + rev: Optional[str] = None, + scm: Optional["Git"] = None, + trie: Optional["GitTrie"] = None, **kwargs: Any, ) -> None: from dvc.scm import resolve_rev diff --git a/dvc/output.py b/dvc/output.py index 60a80ad4a..2bfb98c33 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -290,7 +290,7 @@ class Output: remote=None, repo=None, fs_config=None, - files: List[Dict[str, Any]] = None, + files: Optional[List[Dict[str, Any]]] = None, push: bool = True, ): self.annot = Annotation( diff --git a/dvc/parsing/__init__.py b/dvc/parsing/__init__.py index 363f0fe8b..7e4c7a85f 100644 --- a/dvc/parsing/__init__.py +++ b/dvc/parsing/__init__.py @@ -82,7 +82,7 @@ def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn": def _reraise_err( - exc_cls: Type[Exception], *args, from_exc: Exception = None + exc_cls: Type[Exception], *args, from_exc: Optional[Exception] = None ) -> "NoReturn": err = exc_cls(*args) if from_exc and logger.isEnabledFor(logging.DEBUG): @@ -194,7 +194,7 @@ class DataResolver: def has_key(self, key: str): return self._has_group_and_key(*split_foreach_name(key)) - def _has_group_and_key(self, group: str, key: str = None): + def _has_group_and_key(self, group: str, key: Optional[str] = None): try: definition = self.definitions[group] except KeyError: @@ -236,7 +236,7 @@ class EntryDefinition: self.where = where def _resolve_wdir( - self, context: Context, name: str, wdir: str = None + self, context: Context, name: str, wdir: Optional[str] = None ) -> str: if not wdir: return self.wdir diff --git a/dvc/parsing/context.py b/dvc/parsing/context.py index 0e8a39a07..54046ba3b 100644 --- a/dvc/parsing/context.py +++ b/dvc/parsing/context.py @@ -172,7 +172,7 @@ class Container(Node, ABC): # noqa: B024 return self._convert_with_meta(value, meta) @staticmethod - def _convert_with_meta(value, meta: Meta = None): + def _convert_with_meta(value, meta: Optional[Meta] = None): if value is None or isinstance(value, PRIMITIVES): assert meta return Value(value, meta=meta) @@ -240,7 +240,7 @@ class Container(Node, ABC): # noqa: B024 class CtxList(Container, MutableSequence): _key_transform = staticmethod(int) - def __init__(self, values: Sequence, meta: Meta = None): + def __init__(self, values: Sequence, meta: Optional[Meta] = None): super().__init__(meta=meta) self.data: list = [] self.extend(values) @@ -263,7 +263,12 @@ class CtxList(Container, MutableSequence): class CtxDict(Container, MutableMapping): - def __init__(self, mapping: Mapping = None, meta: Meta = None, **kwargs): + def __init__( + self, + mapping: Optional[Mapping] = None, + meta: Optional[Meta] = None, + **kwargs, + ): super().__init__(meta=meta) self.data: dict = {} @@ -354,7 +359,7 @@ class Context(CtxDict): @classmethod def load_from( - cls, fs, path: str, select_keys: List[str] = None + cls, fs, path: str, select_keys: Optional[List[str]] = None ) -> "Context": from dvc.utils.serialize import load_path @@ -433,8 +438,8 @@ class Context(CtxDict): fs, vars_: List, wdir: str, - stage_name: str = None, - default: str = None, + stage_name: Optional[str] = None, + default: Optional[str] = None, ): if default: to_import = fs.path.join(wdir, default) diff --git a/dvc/render/converter/__init__.py b/dvc/render/converter/__init__.py index dd7018b5a..77e790495 100644 --- a/dvc/render/converter/__init__.py +++ b/dvc/render/converter/__init__.py @@ -5,7 +5,7 @@ class Converter: def __init__( self, plot_id: str, - data: Dict[str, Any] = None, + data: Optional[Dict[str, Any]] = None, properties: Optional[Dict] = None, ): self.plot_id = plot_id diff --git a/dvc/render/converter/vega.py b/dvc/render/converter/vega.py index 879fc5e19..bb203cc5b 100644 --- a/dvc/render/converter/vega.py +++ b/dvc/render/converter/vega.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import Any, Dict, Iterable, List, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from funcy import first, last @@ -106,7 +106,10 @@ class VegaConverter(Converter): """ def __init__( - self, plot_id: str, data: Dict = None, properties: Dict = None + self, + plot_id: str, + data: Optional[Dict] = None, + properties: Optional[Dict] = None, ): super().__init__(plot_id, data, properties) self.plot_id = plot_id @@ -304,8 +307,8 @@ class VegaConverter(Converter): def _update_from_field( target_datapoints: List[Dict], field: str, - source_datapoints: List[Dict] = None, - source_field: str = None, + source_datapoints: Optional[List[Dict]] = None, + source_field: Optional[str] = None, ): if source_datapoints is None: source_datapoints = target_datapoints diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index d4528f1c5..5ecb4ef57 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -92,10 +92,10 @@ class Repo: def _get_repo_dirs( self, - root_dir: str = None, - fs: "FileSystem" = None, + root_dir: Optional[str] = None, + fs: Optional["FileSystem"] = None, uninitialized: bool = False, - scm: "Base" = None, + scm: Optional["Base"] = None, ): from dvc.fs import localfs from dvc.scm import SCM, SCMError @@ -248,7 +248,7 @@ class Repo: return Index.from_repo(self) def check_graph( - self, stages: Iterable["Stage"], callback: Callable = None + self, stages: Iterable["Stage"], callback: Optional[Callable] = None ) -> None: if not getattr(self, "_skip_graph_checks", False): new = self.index.update(stages) diff --git a/dvc/repo/add.py b/dvc/repo/add.py index 7f139bcd0..761aa624b 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -2,7 +2,7 @@ import logging import os from contextlib import contextmanager from itertools import tee -from typing import TYPE_CHECKING, Any, Iterator, List +from typing import TYPE_CHECKING, Any, Iterator, List, Optional import colorama @@ -160,7 +160,7 @@ def add( # noqa: C901 targets: "TargetType", recursive: bool = False, no_commit: bool = False, - fname: str = None, + fname: Optional[str] = None, to_remote: bool = False, **kwargs: Any, ): @@ -251,7 +251,7 @@ def _find_all_targets( def create_stages( repo: "Repo", targets: Iterator[str], - fname: str = None, + fname: Optional[str] = None, transfer: bool = False, external: bool = False, **kwargs: Any, diff --git a/dvc/repo/collect.py b/dvc/repo/collect.py index f51b883cf..3a18f173d 100644 --- a/dvc/repo/collect.py +++ b/dvc/repo/collect.py @@ -1,5 +1,5 @@ import logging -from typing import TYPE_CHECKING, Callable, Iterable, List, Tuple +from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple if TYPE_CHECKING: from dvc.output import Output @@ -14,7 +14,7 @@ StrPaths = List[str] def _collect_outs( - repo: "Repo", output_filter: FilterFn = None, deps: bool = False + repo: "Repo", output_filter: Optional[FilterFn] = None, deps: bool = False ) -> Outputs: index = repo.index index.check_graph() # ensure graph is correct @@ -60,8 +60,8 @@ def _filter_outs( def collect( repo: "Repo", deps: bool = False, - targets: Iterable[str] = None, - output_filter: FilterFn = None, + targets: Optional[Iterable[str]] = None, + output_filter: Optional[FilterFn] = None, recursive: bool = False, duplicates: bool = False, ) -> Tuple[Outputs, StrPaths]: diff --git a/dvc/repo/data.py b/dvc/repo/data.py index c83ef1c08..14a783c87 100644 --- a/dvc/repo/data.py +++ b/dvc/repo/data.py @@ -309,7 +309,7 @@ def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status: def ls( repo: "Repo", - targets: List[Optional[str]] = None, + targets: Optional[List[Optional[str]]] = None, recursive: bool = False, ) -> Iterator[Dict[str, Any]]: targets = targets or [None] diff --git a/dvc/repo/experiments/exceptions.py b/dvc/repo/experiments/exceptions.py index 49856f99d..a81b691f9 100644 --- a/dvc/repo/experiments/exceptions.py +++ b/dvc/repo/experiments/exceptions.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Collection, Iterable +from typing import TYPE_CHECKING, Collection, Iterable, Optional from dvc.exceptions import DvcException, InvalidArgumentError @@ -102,7 +102,10 @@ class UnresolvedExpNamesError(InvalidArgumentError): NAME = "experiment name" def __init__( - self, unresolved_list: Collection[str], *args, git_remote: str = None + self, + unresolved_list: Collection[str], + *args, + git_remote: Optional[str] = None, ): unresolved_names = "; ".join(unresolved_list) if not git_remote: diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 2ddf74d24..1c024cafe 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -308,7 +308,7 @@ class BaseExecutor(ABC): dest_scm: "Git", refs: List[str], force: bool = False, - on_diverged: Callable[[str, bool], None] = None, + on_diverged: Optional[Callable[[str, bool], None]] = None, **kwargs, ) -> Iterable[str]: """Fetch reproduced experiment refs into the specified SCM. @@ -550,7 +550,7 @@ class BaseExecutor(ABC): def _repro_dvc( # noqa: C901 cls, info: "ExecutorInfo", - infofile: str = None, + infofile: Optional[str] = None, log_errors: bool = True, **kwargs, ): diff --git a/dvc/repo/experiments/init.py b/dvc/repo/experiments/init.py index c26172527..cd1d7cde1 100644 --- a/dvc/repo/experiments/init.py +++ b/dvc/repo/experiments/init.py @@ -41,8 +41,10 @@ PROMPTS = { def _prompts( keys: Iterable[str], - defaults: Dict[str, str] = None, - validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None, + defaults: Optional[Dict[str, str]] = None, + validator: Optional[ + Callable[[str, str], Union[str, Tuple[str, str]]] + ] = None, allow_omission: bool = True, stream: Optional[TextIO] = None, ) -> Dict[str, OptStr]: @@ -77,7 +79,9 @@ def _disable_logging(highest_level=logging.CRITICAL): def init_interactive( defaults: Dict[str, str], provided: Dict[str, str], - validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None, + validator: Optional[ + Callable[[str, str], Union[str, Tuple[str, str]]] + ] = None, stream: Optional[TextIO] = None, ) -> Dict[str, str]: command_prompts = lremove(provided.keys(), ["cmd"]) @@ -202,8 +206,8 @@ def init( repo: "Repo", name: str = "train", type: str = "default", # noqa: A002, pylint: disable=redefined-builtin - defaults: Dict[str, str] = None, - overrides: Dict[str, str] = None, + defaults: Optional[Dict[str, str]] = None, + overrides: Optional[Dict[str, str]] = None, interactive: bool = False, force: bool = False, stream: Optional[TextIO] = None, diff --git a/dvc/repo/experiments/utils.py b/dvc/repo/experiments/utils.py index 48e1b3b2f..094d9b838 100644 --- a/dvc/repo/experiments/utils.py +++ b/dvc/repo/experiments/utils.py @@ -205,7 +205,7 @@ def remote_exp_refs_by_baseline( def exp_commits( - scm: "Git", ref_infos: Iterable[ExpRefInfo] = None + scm: "Git", ref_infos: Optional[Iterable[ExpRefInfo]] = None ) -> Iterable[str]: """Iterate over all experiment commits.""" shas: Set["str"] = set() diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 7f9718d1f..07f73b18a 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -9,11 +9,11 @@ from dvc.fs import Schemes from . import locked if TYPE_CHECKING: - from dvc.cloud import Remote + from dvc.data_cloud import Remote from dvc.repo import Repo from dvc.types import TargetType from dvc_data.hashfile.transfer import TransferResult - from dvc_objects.db.base import ObjectDB + from dvc_objects.db import ObjectDB logger = logging.getLogger(__name__) diff --git a/dvc/repo/freeze.py b/dvc/repo/freeze.py index e48818e70..19f7261f2 100644 --- a/dvc/repo/freeze.py +++ b/dvc/repo/freeze.py @@ -10,7 +10,7 @@ if typing.TYPE_CHECKING: def _set(repo: "Repo", target, frozen): stage = repo.stage.get_target(target) stage.frozen = frozen - stage.dvcfile.dump(stage, update_lock=False) + stage.dump(update_lock=False) return stage diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 35110e15f..e98203975 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -35,7 +35,7 @@ def gc( all_commits: bool = False, all_experiments: bool = False, force: bool = False, - jobs: int = None, + jobs: Optional[int] = None, repos: Optional[List[str]] = None, workspace: bool = False, commit_date: Optional[str] = None, diff --git a/dvc/repo/index.py b/dvc/repo/index.py index f8be71510..2c25c486d 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -50,7 +50,7 @@ def log_walk(seq): def collect_files( - repo: "Repo", onerror: Callable[[str, Exception], None] = None + repo: "Repo", onerror: Optional[Callable[[str, Exception], None]] = None ): """Collects all of the stages present in the DVC repo. @@ -114,10 +114,10 @@ class Index: def __init__( self, repo: "Repo", - stages: List["Stage"] = None, - metrics: Dict[str, List[str]] = None, - plots: Dict[str, List[str]] = None, - params: Dict[str, Any] = None, + stages: Optional[List["Stage"]] = None, + metrics: Optional[Dict[str, List[str]]] = None, + plots: Optional[Dict[str, List[str]]] = None, + params: Optional[Dict[str, Any]] = None, ) -> None: self.repo = repo self.stages = stages or [] @@ -134,7 +134,9 @@ class Index: @classmethod def from_repo( - cls, repo: "Repo", onerror: Callable[[str, Exception], None] = None + cls, + repo: "Repo", + onerror: Optional[Callable[[str, Exception], None]] = None, ) -> "Index": stages = [] metrics = {} @@ -315,12 +317,12 @@ class Index: def used_objs( self, - targets: "TargetType" = None, + targets: Optional["TargetType"] = None, with_deps: bool = False, - remote: str = None, + remote: Optional[str] = None, force: bool = False, recursive: bool = False, - jobs: int = None, + jobs: Optional[int] = None, push: bool = False, ) -> "ObjectContainer": from collections import defaultdict diff --git a/dvc/repo/ls.py b/dvc/repo/ls.py index 5adc72c8c..ea4ecc6d0 100644 --- a/dvc/repo/ls.py +++ b/dvc/repo/ls.py @@ -12,8 +12,8 @@ if TYPE_CHECKING: def ls( url: str, path: Optional[str] = None, - rev: str = None, - recursive: bool = None, + rev: Optional[str] = None, + recursive: Optional[bool] = None, dvc_only: bool = False, ): """Methods for getting files and outputs for the repo. @@ -53,7 +53,10 @@ def ls( def _ls( - repo: "Repo", path: str, recursive: bool = None, dvc_only: bool = False + repo: "Repo", + path: str, + recursive: Optional[bool] = None, + dvc_only: bool = False, ): fs: "DVCFileSystem" = repo.dvcfs fs_path = fs.from_os_path(path) diff --git a/dvc/repo/params/show.py b/dvc/repo/params/show.py index 22669fbee..276e34601 100644 --- a/dvc/repo/params/show.py +++ b/dvc/repo/params/show.py @@ -141,7 +141,7 @@ def show( revs=None, targets=None, deps=False, - onerror: Callable = None, + onerror: Optional[Callable] = None, stages=None, hide_workspace=True, ): diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index 5ade3eaf3..693015b8a 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -68,8 +68,8 @@ class Plots: def collect( self, - targets: List[str] = None, - revs: List[str] = None, + targets: Optional[List[str]] = None, + revs: Optional[List[str]] = None, recursive: bool = False, onerror: Optional[Callable] = None, props: Optional[Dict] = None, @@ -180,7 +180,7 @@ class Plots: def show( self, - targets: List[str] = None, + targets: Optional[List[str]] = None, revs=None, props=None, recursive=False, @@ -295,7 +295,7 @@ def _resolve_data_sources(plots_data: Dict): def _collect_plots( repo: "Repo", - targets: List[str] = None, + targets: Optional[List[str]] = None, recursive: bool = False, ) -> Dict[str, Dict]: from dvc.repo.collect import collect @@ -486,7 +486,7 @@ def _collect_definitions( repo: "Repo", targets=None, config_files: Optional[Set[str]] = None, - props: Dict = None, + props: Optional[Dict] = None, onerror: Optional[Callable] = None, **kwargs, ) -> Dict: diff --git a/dvc/repo/pull.py b/dvc/repo/pull.py index 5e51b633a..e1556bb71 100644 --- a/dvc/repo/pull.py +++ b/dvc/repo/pull.py @@ -5,7 +5,7 @@ from dvc.repo import locked from dvc.utils import glob_targets if TYPE_CHECKING: - from dvc_objects.db.base import ObjectDB + from dvc_objects.db import ObjectDB logger = logging.getLogger(__name__) diff --git a/dvc/repo/push.py b/dvc/repo/push.py index cc4d7896a..71786b9a5 100644 --- a/dvc/repo/push.py +++ b/dvc/repo/push.py @@ -6,10 +6,10 @@ from dvc.utils import glob_targets from . import locked if TYPE_CHECKING: - from dvc.cloud import Remote + from dvc.data_cloud import Remote from dvc.repo import Repo from dvc.types import TargetType - from dvc_objects.db.base import ObjectDB + from dvc_objects.db import ObjectDB @locked diff --git a/dvc/repo/scm_context.py b/dvc/repo/scm_context.py index 3dbf02f1d..d8468cdde 100644 --- a/dvc/repo/scm_context.py +++ b/dvc/repo/scm_context.py @@ -9,6 +9,7 @@ from typing import ( Iterable, Iterator, List, + Optional, Set, Union, ) @@ -26,7 +27,9 @@ logger = logging.getLogger(__name__) class SCMContext: - def __init__(self, scm: "Base", config: Dict[str, Any] = None) -> None: + def __init__( + self, scm: "Base", config: Optional[Dict[str, Any]] = None + ) -> None: from funcy import get_in self.scm: "Base" = scm @@ -101,7 +104,7 @@ class SCMContext: @contextmanager def __call__( - self, autostage: bool = None, quiet: bool = None + self, autostage: Optional[bool] = None, quiet: Optional[bool] = None ) -> Iterator["SCMContext"]: try: yield self @@ -147,7 +150,9 @@ class SCMContext: self._cm.__exit__(*exc_args) # pylint: disable=no-member -def scm_context(method, autostage: bool = None, quiet: bool = None): +def scm_context( + method, autostage: Optional[bool] = None, quiet: Optional[bool] = None +): @wraps(method) def run(repo: "Repo", *args, **kw): with repo.scm_context(autostage=autostage, quiet=quiet): diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index f6d8f8a9c..8fe7f63d8 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -103,7 +103,7 @@ class StageLoad: def add( self, single_stage: bool = False, - fname: str = None, + fname: Optional[str] = None, validate: bool = True, force: bool = False, update_lock: bool = False, @@ -133,7 +133,7 @@ class StageLoad: self, single_stage: bool = False, validate: bool = True, - fname: str = None, + fname: Optional[str] = None, force: bool = False, **stage_data, ) -> Union["Stage", "PipelineStage"]: @@ -210,7 +210,9 @@ class StageLoad: path, name = parse_target(target) return self.load_one(path=path, name=name) - def _get_filepath(self, path: str = None, name: str = None) -> str: + def _get_filepath( + self, path: Optional[str] = None, name: Optional[str] = None + ) -> str: if path: return self.repo.fs.path.realpath(path) @@ -230,7 +232,7 @@ class StageLoad: def _get_keys( self, stages: "StageLoader", - name: str = None, + name: Optional[str] = None, accept_group: bool = True, glob: bool = False, ) -> Iterable[str]: @@ -245,8 +247,8 @@ class StageLoad: def load_all( self, - path: str = None, - name: str = None, + path: Optional[str] = None, + name: Optional[str] = None, accept_group: bool = True, glob: bool = False, ) -> StageList: @@ -276,7 +278,9 @@ class StageLoad: keys = self._get_keys(stages, name, accept_group, glob) return [stages[key] for key in keys] - def load_one(self, path: str = None, name: str = None) -> "Stage": + def load_one( + self, path: Optional[str] = None, name: Optional[str] = None + ) -> "Stage": """Load a single stage from a file. Args: @@ -291,20 +295,20 @@ class StageLoad: return stages[name] - def load_file(self, path: str = None) -> StageList: + def load_file(self, path: Optional[str] = None) -> StageList: """Load all of the stages from a file.""" return self.load_all(path) - def load_glob(self, path: str, expr: str = None): + def load_glob(self, path: str, expr: Optional[str] = None): """Load stages from `path`, filtered with `expr` provided.""" return self.load_all(path, expr, glob=True) def collect( self, - target: str = None, + target: Optional[str] = None, with_deps: bool = False, recursive: bool = False, - graph: "DiGraph" = None, + graph: Optional["DiGraph"] = None, glob: bool = False, ) -> StageIter: """Collect list of stages from the provided target. @@ -351,10 +355,10 @@ class StageLoad: def collect_granular( self, - target: str = None, + target: Optional[str] = None, with_deps: bool = False, recursive: bool = False, - graph: "DiGraph" = None, + graph: Optional["DiGraph"] = None, ) -> List[StageInfo]: """Collects a list of (stage, filter_info) from the given target. diff --git a/dvc/repo/worktree.py b/dvc/repo/worktree.py index f2c032642..1514903c6 100644 --- a/dvc/repo/worktree.py +++ b/dvc/repo/worktree.py @@ -5,10 +5,10 @@ from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union, cast from dvc.fs.callbacks import Callback if TYPE_CHECKING: - from dvc.cloud import Remote - from dvc.index import Index, IndexView + from dvc.data_cloud import Remote from dvc.output import Output from dvc.repo import Repo + from dvc.repo.index import Index, IndexView from dvc.repo.stage import StageInfo from dvc.stage import Stage from dvc.types import TargetType @@ -144,7 +144,7 @@ def push_worktree( for out in stage.outs: workspace, _key = out.index_key _update_out_meta(out, repo.index.data[workspace]) - stage.dvcfile.dump(stage, with_files=True, update_pipeline=False) + stage.dump(with_files=True, update_pipeline=False) return pushed @@ -231,7 +231,7 @@ def update_worktree_stages( stage.save(merge_versioned=True) for out in stage.outs: _update_out_meta(out, remote_index) - stage.dvcfile.dump(stage, with_files=True, update_pipeline=False) + stage.dump(with_files=True, update_pipeline=False) def _fetch_out_changes( diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 6fa16d0b8..4937eb67f 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -38,7 +38,7 @@ from .utils import ( if TYPE_CHECKING: from dvc.dvcfile import ProjectFile, SingleStageFile from dvc_data.hashfile.hash_info import HashInfo - from dvc_objects.db.base import ObjectDB + from dvc_objects.db import ObjectDB logger = logging.getLogger(__name__) # Disallow all punctuation characters except hyphen and underscore @@ -396,7 +396,7 @@ class Stage(params.StageParams): def transfer( self, source: str, - odb: "ObjectDB" = None, + odb: Optional["ObjectDB"] = None, to_remote: bool = False, **kwargs: Any, ) -> None: diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index d2da7b9f9..7e3bf7adc 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -1,6 +1,6 @@ import os import pathlib -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Optional, Union from funcy import concat, first, lsplit, rpartial @@ -278,7 +278,9 @@ def check_stage_exists( ) -def validate_kwargs(single_stage: bool = False, fname: str = None, **kwargs): +def validate_kwargs( + single_stage: bool = False, fname: Optional[str] = None, **kwargs +): """Prepare, validate and process kwargs passed from cli""" cmd = kwargs.get("cmd") if not cmd and not single_stage: diff --git a/dvc/ui/__init__.py b/dvc/ui/__init__.py index a6d412617..0e343c657 100644 --- a/dvc/ui/__init__.py +++ b/dvc/ui/__init__.py @@ -26,7 +26,9 @@ if TYPE_CHECKING: class Formatter: - def __init__(self, theme: Dict = None, defaults: Dict = None) -> None: + def __init__( + self, theme: Optional[Dict] = None, defaults: Optional[Dict] = None + ) -> None: from collections import defaultdict theme = theme or { @@ -36,7 +38,9 @@ class Formatter: } self.theme = defaultdict(lambda: defaults or {}, theme) - def format(self, message: str, style: str = None, **kwargs) -> str: + def format( + self, message: str, style: Optional[str] = None, **kwargs + ) -> str: from dvc.utils import colorize return colorize(message, **self.theme[style]) @@ -44,7 +48,7 @@ class Formatter: class Console: def __init__( - self, formatter: Formatter = None, enable: bool = False + self, formatter: Optional[Formatter] = None, enable: bool = False ) -> None: from contextvars import ContextVar @@ -69,9 +73,9 @@ class Console: def error_write( self, *objects: Any, - style: str = None, - sep: str = None, - end: str = None, + style: Optional[str] = None, + sep: Optional[str] = None, + end: Optional[str] = None, styled: bool = False, force: bool = True, ) -> None: @@ -88,8 +92,8 @@ class Console: def write_json( self, data: Any, - indent: int = None, - highlight: bool = None, + indent: Optional[int] = None, + highlight: Optional[bool] = None, stderr: bool = False, skip_keys: bool = False, ensure_ascii: bool = True, @@ -119,13 +123,13 @@ class Console: def write( self, *objects: Any, - style: str = None, - sep: str = None, - end: str = None, + style: Optional[str] = None, + sep: Optional[str] = None, + end: Optional[str] = None, stderr: bool = False, force: bool = False, styled: bool = False, - file: TextIO = None, + file: Optional[TextIO] = None, ) -> None: import sys @@ -172,7 +176,10 @@ class Console: self._paginate.reset(tok) def prompt( - self, text: str, choices: Iterable[str] = None, password: bool = False + self, + text: str, + choices: Optional[Iterable[str]] = None, + password: bool = False, ) -> Optional[str]: while True: try: @@ -222,13 +229,15 @@ class Console: def table( self, data: "TableData", - headers: "Headers" = None, + headers: Optional["Headers"] = None, markdown: bool = False, rich_table: bool = False, force: bool = True, pager: bool = False, - header_styles: Union[Dict[str, "Styles"], Sequence["Styles"]] = None, - row_styles: Sequence["Styles"] = None, + header_styles: Optional[ + Union[Dict[str, "Styles"], Sequence["Styles"]] + ] = None, + row_styles: Optional[Sequence["Styles"]] = None, borders: Union[bool, str] = False, ) -> None: from dvc.ui import table as t diff --git a/dvc/ui/table.py b/dvc/ui/table.py index a6e0655ed..de55c9b44 100644 --- a/dvc/ui/table.py +++ b/dvc/ui/table.py @@ -1,7 +1,7 @@ from collections import abc from contextlib import ExitStack, contextmanager from itertools import zip_longest -from typing import TYPE_CHECKING, Dict, Iterator, Sequence, Union +from typing import TYPE_CHECKING, Dict, Iterator, Optional, Sequence, Union from dvc.types import DictStrAny @@ -24,7 +24,7 @@ Styles = DictStrAny def plain_table( ui: "Console", data: TableData, - headers: Headers = None, + headers: Optional[Headers] = None, markdown: bool = False, pager: bool = False, force: bool = True, @@ -76,10 +76,10 @@ def console_width( def rich_table( ui: "Console", data: TableData, - headers: Headers = None, + headers: Optional[Headers] = None, pager: bool = False, - header_styles: Union[Dict[str, Styles], Sequence[Styles]] = None, - row_styles: Sequence[Styles] = None, + header_styles: Optional[Union[Dict[str, Styles], Sequence[Styles]]] = None, + row_styles: Optional[Sequence[Styles]] = None, borders: Union[bool, str] = False, ) -> None: from rich import box diff --git a/dvc/updater.py b/dvc/updater.py index 07c4026d7..96b7749bf 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -123,7 +123,7 @@ class Updater: def _get_message( self, latest: str, - current: str = None, + current: Optional[str] = None, color: str = "yellow", pkg: Optional[str] = None, ) -> "RichText": diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index ae57b541a..6c4f15ebf 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -331,7 +331,7 @@ def error_link(name): def parse_target( - target: str, default: str = None, isa_glob: bool = False + target: str, default: Optional[str] = None, isa_glob: bool = False ) -> Tuple[Optional[str], Optional[str]]: from dvc.dvcfile import LOCK_FILE, PROJECT_FILE, is_valid_filename from dvc.exceptions import DvcException diff --git a/dvc/utils/pkg.py b/dvc/utils/pkg.py index 82f8a29ed..3f507a568 100644 --- a/dvc/utils/pkg.py +++ b/dvc/utils/pkg.py @@ -1,5 +1,6 @@ try: # file is created during dvc build - from .build import PKG # pylint:disable=unused-import + # pylint:disable=unused-import + from .build import PKG # type: ignore[import] except ImportError: PKG = None # type: ignore[assignment] diff --git a/dvc/utils/serialize/_common.py b/dvc/utils/serialize/_common.py index 606393531..349d74593 100644 --- a/dvc/utils/serialize/_common.py +++ b/dvc/utils/serialize/_common.py @@ -7,6 +7,7 @@ from typing import ( Callable, ContextManager, Dict, + Optional, Protocol, Union, ) @@ -22,20 +23,22 @@ if TYPE_CHECKING: class DumperFn(Protocol): def __call__( - self, path: "StrPath", data: Any, fs: "FileSystem" = None + self, path: "StrPath", data: Any, fs: Optional["FileSystem"] = None ) -> Any: ... class ModifierFn(Protocol): def __call__( - self, path: "StrPath", fs: "FileSystem" = None + self, path: "StrPath", fs: Optional["FileSystem"] = None ) -> ContextManager[Dict]: ... class LoaderFn(Protocol): - def __call__(self, path: "StrPath", fs: "FileSystem" = None) -> Any: + def __call__( + self, path: "StrPath", fs: Optional["FileSystem"] = None + ) -> Any: ... @@ -62,7 +65,9 @@ class EncodingError(ParseError): super().__init__(path, f"is not valid {encoding}") -def _load_data(path: "StrPath", parser: ParserFn, fs: "FileSystem" = None): +def _load_data( + path: "StrPath", parser: ParserFn, fs: Optional["FileSystem"] = None +): open_fn = fs.open if fs else open encoding = "utf-8" with open_fn(path, encoding=encoding) as fd: @@ -74,7 +79,7 @@ def _dump_data( path, data: Any, dumper: DumperFn, - fs: "FileSystem" = None, + fs: Optional["FileSystem"] = None, **dumper_args, ): open_fn = fs.open if fs else open @@ -87,7 +92,7 @@ def _modify_data( path: "StrPath", parser: ParserFn, dumper: DumperFn, - fs: "FileSystem" = None, + fs: Optional["FileSystem"] = None, ): exists_fn = fs.exists if fs else os.path.exists file_exists = exists_fn(path) diff --git a/dvc/utils/strictyaml.py b/dvc/utils/strictyaml.py index 02176605c..e412a60ba 100644 --- a/dvc/utils/strictyaml.py +++ b/dvc/utils/strictyaml.py @@ -101,7 +101,7 @@ class YAMLSyntaxError(PrettyDvcException, YAMLFileCorruptedError): return f"in line {mark.line + 1}, column {mark.column + 1}" def prepare_message( - message: str, mark: "StreamMark" = None + message: str, mark: Optional["StreamMark"] = None ) -> "RichText": cause = ", ".join( [message.capitalize(), prepare_linecol(mark) if mark else ""] @@ -196,9 +196,9 @@ class YAMLValidationError(PrettyDvcException): def __init__( self, exc: "MultipleInvalid", - path: str = None, - text: str = None, - rev: str = None, + path: Optional[str] = None, + text: Optional[str] = None, + rev: Optional[str] = None, ) -> None: self.text = text or "" self.exc = exc @@ -263,9 +263,9 @@ class YAMLValidationError(PrettyDvcException): def validate( data: _T, schema: Callable[[_T], _T], - text: str = None, - path: str = None, - rev: str = None, + text: Optional[str] = None, + path: Optional[str] = None, + rev: Optional[str] = None, ) -> _T: from voluptuous import MultipleInvalid @@ -277,8 +277,8 @@ def validate( def load( path: str, - schema: Callable[[_T], _T] = None, - fs: "FileSystem" = None, + schema: Optional[Callable[[_T], _T]] = None, + fs: Optional["FileSystem"] = None, encoding: str = "utf-8", round_trip: bool = False, ) -> Any: diff --git a/dvc/utils/threadpool.py b/dvc/utils/threadpool.py index dde15ba6e..97de574d8 100644 --- a/dvc/utils/threadpool.py +++ b/dvc/utils/threadpool.py @@ -2,7 +2,7 @@ import queue import sys from concurrent import futures from itertools import islice -from typing import Any, Callable, Iterable, Iterator, Set, TypeVar +from typing import Any, Callable, Iterable, Iterator, Optional, Set, TypeVar _T = TypeVar("_T") @@ -11,7 +11,10 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor): _max_workers: int def __init__( - self, max_workers: int = None, cancel_on_error: bool = False, **kwargs + self, + max_workers: Optional[int] = None, + cancel_on_error: bool = False, + **kwargs ): super().__init__(max_workers=max_workers, **kwargs) self._cancel_on_error = cancel_on_error diff --git a/pyproject.toml b/pyproject.toml index 6701ecb21..5ebb91b78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,10 @@ tests = [ "types-requests", "types-tabulate", "types-toml", + "types-appdirs", + "types-colorama", + "types-psutil", + "types-tqdm", ] all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"] dev = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs,tests]"] @@ -221,16 +225,50 @@ show_error_codes = true show_error_context = true show_traceback = true pretty = true -# See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports. -ignore_missing_imports = true -implicit_optional = true +no_implicit_optional = true check_untyped_defs = false +strict_equality = true +strict_concatenate = true # Warnings warn_no_return = true warn_redundant_casts = true +warn_unused_configs = true warn_unreachable = true files = ["dvc"] +[[tool.mypy.overrides]] +module = [ + "celery.*", + "configobj.*", + "dpath.*", + "dvc_data.*", + "dvc_http.*", + "dvc_objects.*", + "dvc_render.*", + "dvc_ssh", + "dvc_studio_client.*", + "dvc_task.*", + "flatten_dict", + "fsspec.*", + "funcy.*", # https://github.com/Suor/funcy/issues/106, + "grandalf.*", + "ipdb", + "iterative_telemetry", + "kombu.*", + "networkx.*", # https://github.com/networkx/networkx/issues/3988 + "pygtrie.*", + "pyinstrument", + "pytest_docker.plugin", + "ruamel.yaml.*", + "scmrepo.*", + "tpi.*", + "viztracer", + "voluptuous", + "yappi", + "zc.*", +] +ignore_missing_imports = true + [tool.pylint.master] extension-pkg-whitelist = ["pygit2"] init-hook = "import sys; sys.path.append(str('tests'))"