From 388eccad13fc4622ba0d7674a702d0071790b890 Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Fri, 8 Jan 2021 19:50:52 +0545 Subject: [PATCH] fix mypy errors; add mypy precommit hooks (#5237) --- .pre-commit-config.yaml | 6 ++++++ dvc/repo/add.py | 18 ++++++++++++++---- dvc/repo/params/show.py | 3 ++- dvc/repo/plots/__init__.py | 11 ++++++----- dvc/stage/__init__.py | 5 +++-- dvc/types.py | 3 ++- dvc/utils/collections.py | 9 +++++++++ setup.cfg | 2 +- 8 files changed, 43 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bc3c4ba1..55a11e221 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,12 @@ repos: - flake8-string-format repo: https://gitlab.com/pycqa/flake8 rev: 3.8.4 + - hooks: + - id: mypy + language_version: python3 + files: ^dvc/ + repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.790' - repo: local hooks: - id: pylint diff --git a/dvc/repo/add.py b/dvc/repo/add.py index ea1f39b31..5e59114f5 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -1,5 +1,6 @@ import logging import os +from typing import TYPE_CHECKING import colorama @@ -14,20 +15,29 @@ from ..repo.scm_context import scm_context from ..utils import LARGE_DIR_SIZE, glob_targets, resolve_paths from . import locked +if TYPE_CHECKING: + from dvc.types import TargetType + + logger = logging.getLogger(__name__) @locked @scm_context def add( - repo, targets, recursive=False, no_commit=False, fname=None, **kwargs, + repo, + targets: "TargetType", + recursive=False, + no_commit=False, + fname=None, + **kwargs, ): + from dvc.utils.collections import ensure_list + if recursive and fname: raise RecursiveAddingWhileUsingFilename() - if isinstance(targets, str): - targets = [targets] - + targets = ensure_list(targets) link_failures = [] stages_list = [] num_targets = len(targets) diff --git a/dvc/repo/params/show.py b/dvc/repo/params/show.py index e0084488a..a400a8620 100644 --- a/dvc/repo/params/show.py +++ b/dvc/repo/params/show.py @@ -14,6 +14,7 @@ from dvc.utils.serialize import LOADERS, ParseError if TYPE_CHECKING: from dvc.output.base import BaseOutput from dvc.repo import Repo + from dvc.types import DvcPath logger = logging.getLogger(__name__) @@ -26,7 +27,7 @@ def _is_params(dep: "BaseOutput"): return isinstance(dep, ParamsDependency) -def _collect_configs(repo: "Repo", rev, targets=None) -> List[PathInfo]: +def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]: params, path_infos = collect( repo, targets=targets or [], diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index 9b752f329..b85817f9e 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -50,9 +50,10 @@ class Plots: Data parsing is postponed, since it's affected by props. """ from dvc.tree.repo import RepoTree + from dvc.utils.collections import ensure_list - targets = [targets] if isinstance(targets, str) else targets or [] - data = {} + targets = ensure_list(targets) + data: Dict[str, Dict] = {} for rev in self.repo.brancher(revs=revs): # .brancher() adds unwanted workspace if revs is not None and rev not in revs: @@ -121,18 +122,18 @@ class Plots: def show( self, - targets=None, + targets: List[str] = None, revs=None, props=None, templates=None, recursive=False, ): + from dvc.utils.collections import ensure_list data = self.collect(targets, revs, recursive) # If any mentioned plot doesn't have any data then that's an error - targets = [targets] if isinstance(targets, str) else targets or [] - for target in targets: + for target in ensure_list(targets): rpath = relpath(target, self.repo.root_dir) if not any( "data" in rev_data[key] diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 101aaef3f..a3c0ee4f7 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -37,6 +37,7 @@ from .utils import ( logger = logging.getLogger(__name__) # Disallow all punctuation characters except hyphen and underscore INVALID_STAGENAME_CHARS = set(string.punctuation) - {"_", "-"} +Env = Dict[str, str] def loads_from(cls, repo, path, wdir, data): @@ -252,8 +253,8 @@ class Stage(params.StageParams): return any(out.checkpoint for out in self.outs) @property - def env(self) -> Dict[str, str]: - env = {} + def env(self) -> Env: + env: Env = {} for out in self.outs: if any(out.env.keys() and env.keys()): raise DvcException("Duplicated env variable") diff --git a/dvc/types.py b/dvc/types.py index f38b1727e..837e5cd52 100644 --- a/dvc/types.py +++ b/dvc/types.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Union from dvc.path_info import PathInfo, URLInfo @@ -10,3 +10,4 @@ DvcPath = Union[PathInfo, URLInfo] AnyPath = Union[str, DvcPath, StrPath] OptStr = Optional[str] +TargetType = Union[List[str], str] diff --git a/dvc/utils/collections.py b/dvc/utils/collections.py index d59869b61..8993b9f4e 100644 --- a/dvc/utils/collections.py +++ b/dvc/utils/collections.py @@ -1,5 +1,6 @@ import os from collections.abc import Mapping +from typing import Iterable, List, Union from pygtrie import StringTrie as _StringTrie @@ -57,3 +58,11 @@ def apply_diff(src, dest): src.__class__.__name__, dest.__class__.__name__ ) ) + + +def ensure_list(item: Union[Iterable[str], str, None]) -> List[str]: + if item is None: + return [] + if isinstance(item, str): + return [item] + return list(item) diff --git a/setup.cfg b/setup.cfg index 5642f6154..b6eb8473c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,5 +40,5 @@ check_untyped_defs = False # Warnings warn_no_return=True warn_redundant_casts=True -warn_unused_ignores=True warn_unreachable = True +files = dvc