diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0c3a8f79d..4f7987443 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -30,7 +30,7 @@ jobs: - name: Install requirements run: | pip install wheel - pip install ".[all,tests,terraform]" Pygments collective.checkdocs pre-commit + pip install ".[all,tests,terraform]" pre-commit - name: install temporary dependencies run: | pip install git+https://github.com/isidentical/gdrivefs@service-account diff --git a/.gitignore b/.gitignore index 30fd2dc95..6acd91491 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ build/ dist/ *.egg-info/ +.eggs hooks/*.pyc diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index ea1bd51eb..000000000 --- a/.pylintrc +++ /dev/null @@ -1,615 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-whitelist=pygit2 - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -init-hook="import sys; - from pathlib import Path; - from pylint.config import find_pylintrc; - - tests_folder = Path(find_pylintrc()).parent / 'tests' - sys.path.append(str(tests_folder)); - " - - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins=pylint_pytest,pylint_plugin_disable - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=format, - # ^ most of them are covered by flake8, - # and are conventions rather than warnings or errors - refactoring, # will be noisy if it starts giving suggestions - spelling, - # design is too-few-* and too-many-* checks, too-much-* for us right now - design, - # following were generated by default - # most of them are redundant Py3 checks for Py2+3 codebase - print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, - # custom disables start here - no-self-use, - invalid-name, - # already a check enabled for `wildcard-import` - unused-wildcard-import, - misplaced-comparison-constant, - # probably soon: https://github.com/iterative/dvc/issues/1843 - logging-format-interpolation, - logging-fstring-interpolation, - # we don't have proper docstrings yet - # even if we do, we'll be doing it gradually - missing-function-docstring, - missing-module-docstring, - missing-class-docstring, - duplicate-code, - # too noisy, and detects issues even when import is inside a function - # see: https://github.com/PyCQA/pylint/issues/850 - cyclic-import, - # isort should cover these: - wrong-import-order, - wrong-import-position, - ungrouped-imports, - multiple-imports, - # we use it to speedup/optimize - import-outside-toplevel, - fixme, - # we use cause of the exception for error message output - # so some of the times we don't use `raise from` to not show the cause - # of the error - raise-missing-from, - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member, - no-else-return, - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[LOGGING] - -# Format style used to check logging format string. `old` means using % -# formatting, while `new` is for `{}` formatting. -logging-format-style=old - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package.. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members=hdfs.*,pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules=azure - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - -signature-mutators=funcy.decorators.decorator, - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_|args|kwargs - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=79 - -# Maximum number of lines in a module. -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. -#class-attribute-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - ex, - Run, - _, - s3, - gs, - x, y, - x0, y0, - x1, y1, - dx, dy - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. -#variable-rgx= - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[DESIGN] - -# Maximum number of arguments for function / method. -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in an if statement. -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception". -overgeneral-exceptions=Exception diff --git a/dvc/info.py b/dvc/info.py index 577934b95..06340be99 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -15,17 +15,15 @@ from dvc.utils import error_link from dvc.utils.pkg import PKG from dvc.version import __version__ -if PKG is None: - package = "" -else: - package = f"({PKG})" - try: import importlib.metadata as importlib_metadata except ImportError: # < 3.8 import importlib_metadata # type: ignore[no-redef] +package = "" if PKG is None else f"({PKG})" + + def get_dvc_info(): info = [ f"DVC version: {__version__} {package}", diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 126a4409a..d7d977ca7 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -490,7 +490,7 @@ class BaseExecutor(ABC): push_cache=push_cache, run_cache=run_cache, ) - except BaseException as exc: + except BaseException as exc: # pylint: disable=broad-except logger.warning( "Something went wrong while auto pushing experiment " f"to the remote '{git_remote}': {exc}" diff --git a/dvc/utils/pkg.py b/dvc/utils/pkg.py index 230b9e798..970dad092 100644 --- a/dvc/utils/pkg.py +++ b/dvc/utils/pkg.py @@ -2,4 +2,4 @@ try: # file is created during dvc build from .build import PKG # noqa, pylint:disable=unused-import except ImportError: - PKG = None + PKG = None # type: ignore[assignment] diff --git a/pyproject.toml b/pyproject.toml index 1d05edf15..c507b8f01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,6 @@ [build-system] -requires = [ - "setuptools>=58.2", - "wheel", - "setuptools_scm[toml]>=6.3.1", - # Prevents pkg_resources import in entry point script, - # see https://github.com/ninjaaron/fast-entry_points. - # This saves about 200 ms on startup time for non-wheel installs. - "fastentrypoints>=0.12" -] +requires = ["setuptools>=48", "wheel", "setuptools_scm[toml]>=6.3.1"] +build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "dvc/_version.py" @@ -29,3 +22,58 @@ exclude = ''' | dist )/ ''' + +[tool.isort] +profile = "black" +known_first_party = ["dvc", "tests"] +line_length = 79 + +[tool.pytest.ini_options] +log_level = "debug" +addopts = "-ra" +markers = [ + "needs_internet: Might need network access for the tests", +] + +[tool.mypy] +# Error output +show_column_numbers = true +show_error_codes = true +show_error_context = true +show_traceback = true +pretty = true +# See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports. +ignore_missing_imports = true +check_untyped_defs = false +# Warnings +warn_no_return = true +warn_redundant_casts = true +warn_unreachable = true +files = ["dvc"] + +[tool.pylint.master] +extension-pkg-whitelist = ["pygit2"] +init-hook = "import sys; sys.path.append(str('tests'))" +load-plugins = ["pylint_pytest", "pylint_plugin_disable"] + +[tool.pylint.message_control] +disable = [ + "format", "refactoring", "spelling", "design", "no-self-use", + "invalid-name", "misplaced-comparison-constant", "duplicate-code", "fixme", + "unused-wildcard-import", "cyclic-import", "wrong-import-order", + "wrong-import-position", "ungrouped-imports", "multiple-imports", + "logging-format-interpolation", "logging-fstring-interpolation", + "missing-function-docstring", "missing-module-docstring", + "missing-class-docstring", "raise-missing-from", "import-outside-toplevel", +] +enable = ["c-extension-no-member", "no-else-return"] + +[tool.pylint.typecheck] +generated-members = ["pytest.lazy_fixture", "logging.TRACE", "logger.trace", "sys.getwindowsversion"] +ignored-classes = ["Dvcfile"] +ignored-modules = ["azure"] +signature-mutators = ["funcy.decorators.decorator"] + +[tool.pylint.variables] +dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" +ignored-argument-names = "_.*|^ignored_|^unused_|args|kwargs" diff --git a/setup.cfg b/setup.cfg index 45e8188cc..69244c457 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,14 +26,12 @@ classifiers = [options] setup_requires = - setuptools>=58.2 + setuptools>=48 setuptools_scm[toml]>=6.3.1 - fastentrypoints>=0.12 python_requires = >=3.7 zip_safe = False packages = find: include_package_data = True -use_scm_version = True install_requires = # See https://github.com/pyinstaller/pyinstaller/issues/1945 ply>=3.9 @@ -42,7 +40,6 @@ install_requires = gitpython>3 dulwich>=0.20.23 pygit2>=1.5.0 - setuptools>=34.0.0 nanotime>=0.5.2 pyasn1>=0.4.1 voluptuous>=0.11.7 @@ -173,37 +170,8 @@ ignore= W503, # Line break occurred before a binary operator P1, # unindexed parameters in the str.format, see: # https://pypi.org/project/flake8-string-format/ -max_line_length=79 +max_line_length = 79 max-complexity = 15 -select=B,C,E,F,W,T4,B902,T,P -show_source=true -count=true - -[isort] -profile=black -known_first_party=dvc,tests -line_length=79 - -[tool:pytest] -log_level = debug -addopts = -ra -markers = - needs_internet: Might need network access for the tests - -[mypy] -# Error output -show_column_numbers = True -show_error_codes = True -show_error_context = True -show_traceback = True -pretty = True - -# See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports. -ignore_missing_imports = True -check_untyped_defs = False - -# Warnings -warn_no_return=True -warn_redundant_casts=True -warn_unreachable = True -files = dvc +select = B,C,E,F,W,T4,B902,T,P +show_source = true +count = true