From cbc111b51937762308170bf18c59da582fe63cde Mon Sep 17 00:00:00 2001 From: skshetry <18718008+skshetry@users.noreply.github.com> Date: Thu, 9 Nov 2023 21:26:37 +0545 Subject: [PATCH] remove flake8/bandit (#10082) ruff already runs most of the flake8/bandit checks --- .flake8 | 21 ------------------- .pre-commit-config.yaml | 14 ------------- dvc/commands/completion.py | 2 +- dvc/commands/dag.py | 2 +- dvc/commands/experiments/show.py | 2 +- dvc/daemon.py | 8 +++---- dvc/env.py | 2 +- dvc/fs/dvc.py | 2 +- dvc/lock.py | 2 +- dvc/logger.py | 2 +- dvc/output.py | 2 +- dvc/parsing/interpolate.py | 2 +- dvc/repo/__init__.py | 2 +- dvc/repo/artifacts.py | 2 +- dvc/repo/brancher.py | 2 +- dvc/repo/experiments/__init__.py | 2 +- dvc/repo/experiments/apply.py | 2 +- dvc/repo/experiments/executor/base.py | 6 +++--- dvc/repo/experiments/queue/base.py | 6 +++--- dvc/repo/fetch.py | 2 +- dvc/repo/imp_url.py | 2 +- dvc/repo/open_repo.py | 2 +- dvc/repo/push.py | 2 +- dvc/stage/__init__.py | 6 ++---- dvc/stage/loader.py | 2 +- dvc/stage/run.py | 4 ++-- .../cli/stories/test_modify_data.py | 4 ++-- dvc/testing/benchmarks/fixtures.py | 4 ++-- dvc/testing/fixtures.py | 6 +++--- dvc/ui/pager.py | 2 +- pyproject.toml | 6 ------ 31 files changed, 41 insertions(+), 84 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index edf22d70b..000000000 --- a/.flake8 +++ /dev/null @@ -1,21 +0,0 @@ -[flake8] -max_line_length = 88 -extend-ignore= - E501 - # Whitespace before ':' - E203 - # Line break occurred before a binary operator - W503 - # unindexed parameters in the str.format, see: - # https://pypi.org/project/flake8-string-format/ - P1 -max-complexity = 10 -show_source = true -count = true -min_python_version = 3.8.0 -per-file-ignores = - dvc/testing/benchmarks/conftest.py:F401 - dvc/testing/conftest.py:F401 - dvc/testing/plugin.py:F401 - tests/conftest.py:F401 - tests/func/experiments/executor/test_ssh.py:F401 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99d9789e0..3a73ae2fa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,20 +32,6 @@ repos: hooks: - id: codespell additional_dependencies: ["tomli"] - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - additional_dependencies: - - flake8-string-format - - flake8-typing-imports - - repo: https://github.com/PyCQA/bandit - rev: '1.7.5' - hooks: - - id: bandit - args: ["-c", "pyproject.toml", "--quiet", "--format=custom"] - files: ^dvc/ - additional_dependencies: [".[toml]"] - repo: local hooks: - id: mypy diff --git a/dvc/commands/completion.py b/dvc/commands/completion.py index ac2dc0721..98401ca9a 100644 --- a/dvc/commands/completion.py +++ b/dvc/commands/completion.py @@ -18,7 +18,7 @@ class CmdCompletion(CmdBaseNoRepo): shell = self.args.shell parser = self.args.parser - script = shtab.complete(parser, shell=shell, preamble=PREAMBLE) # nosec B604 + script = shtab.complete(parser, shell=shell, preamble=PREAMBLE) ui.write(script, force=True) return 0 diff --git a/dvc/commands/dag.py b/dvc/commands/dag.py index 1e8d76a99..6ae327dc4 100644 --- a/dvc/commands/dag.py +++ b/dvc/commands/dag.py @@ -86,7 +86,7 @@ def _collect_targets(repo, target, outs): targets.extend([str(out) for out in stage.outs]) continue - for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)): # noqa: B301 + for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)): targets.extend(str(out)) return targets diff --git a/dvc/commands/experiments/show.py b/dvc/commands/experiments/show.py index cfb8b49b9..524c2145a 100644 --- a/dvc/commands/experiments/show.py +++ b/dvc/commands/experiments/show.py @@ -54,7 +54,7 @@ def baseline_styler(typ): return {"style": "bold"} if typ == "baseline" else {} -def show_experiments( # noqa: C901 +def show_experiments( td: "TabularData", headers: Dict[str, Iterable[str]], keep=None, diff --git a/dvc/daemon.py b/dvc/daemon.py index fe6c518c3..c90f9d5a3 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -3,7 +3,7 @@ import inspect import logging import os -import subprocess # nosec B404 +import subprocess import sys from contextlib import nullcontext from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Sequence, Union @@ -28,7 +28,7 @@ def _suppress_resource_warning(popen: subprocess.Popen) -> None: def _win_detached_subprocess(args: Sequence[str], **kwargs) -> int: assert os.name == "nt" - from subprocess import ( # type: ignore[attr-defined] # nosec B404 + from subprocess import ( # type: ignore[attr-defined] CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, STARTF_USESHOWWINDOW, @@ -44,7 +44,7 @@ def _win_detached_subprocess(args: Sequence[str], **kwargs) -> int: popen = subprocess.Popen( args, close_fds=True, - shell=False, # noqa: S603 # nosec B603 + shell=False, # noqa: S603 startupinfo=startupinfo, creationflags=creationflags, **kwargs, @@ -108,7 +108,7 @@ def _posix_detached_subprocess(args: Sequence[str], **kwargs) -> int: proc = subprocess.Popen( args, - shell=False, # noqa: S603 # nosec B603 + shell=False, # noqa: S603 close_fds=True, **kwargs, ) diff --git a/dvc/env.py b/dvc/env.py index 919b2aefd..6f481f622 100644 --- a/dvc/env.py +++ b/dvc/env.py @@ -14,7 +14,7 @@ DVC_SHOW_TRACEBACK = "DVC_SHOW_TRACEBACK" DVC_SITE_CACHE_DIR = "DVC_SITE_CACHE_DIR" DVC_STUDIO_OFFLINE = "DVC_STUDIO_OFFLINE" DVC_STUDIO_REPO_URL = "DVC_STUDIO_REPO_URL" -DVC_STUDIO_TOKEN = "DVC_STUDIO_TOKEN" # noqa: S105 # nosec B105 +DVC_STUDIO_TOKEN = "DVC_STUDIO_TOKEN" # noqa: S105 DVC_STUDIO_URL = "DVC_STUDIO_URL" DVC_SYSTEM_CONFIG_DIR = "DVC_SYSTEM_CONFIG_DIR" DVC_UPDATER_ENDPOINT = "DVC_UPDATER_ENDPOINT" diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index d6c9865e4..6397a9229 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -303,7 +303,7 @@ class _DVCFileSystem(AbstractFileSystem): except FileNotFoundError: return False - def ls(self, path, detail=True, dvc_only=False, **kwargs): # noqa: C901 + def ls(self, path, detail=True, dvc_only=False, **kwargs): key = self._get_key_from_relative(path) repo, dvc_fs, subkey = self._get_subrepo_info(key) diff --git a/dvc/lock.py b/dvc/lock.py index 9a58c090a..5e30b8dbe 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -185,7 +185,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase): if self._tmp_dir is not None: # Under Windows file path length is limited so we hash it - hasher = hashlib.md5( # nosec B324, B303 # noqa: S324 + hasher = hashlib.md5( # noqa: S324 self._claimfile.encode() ) filename = hasher.hexdigest() diff --git a/dvc/logger.py b/dvc/logger.py index da4a4a650..41c2e5a52 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -151,7 +151,7 @@ class LoggerHandler(logging.StreamHandler): if not _is_verbose(): return # pylint: disable-next=broad-except - except Exception: # noqa: BLE001, S110 # nosec B110 + except Exception: # noqa: BLE001, S110 pass msg = self.format(record) diff --git a/dvc/output.py b/dvc/output.py index 0e1fdd771..cf67285bc 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -1157,7 +1157,7 @@ class Output: return obj.filter(prefix) return obj - def get_used_objs( # noqa: C901, PLR0911 + def get_used_objs( # noqa: PLR0911 self, **kwargs ) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]: """Return filtered set of used object IDs for this out.""" diff --git a/dvc/parsing/interpolate.py b/dvc/parsing/interpolate.py index 4b9b0686c..1d2a4d2d1 100644 --- a/dvc/parsing/interpolate.py +++ b/dvc/parsing/interpolate.py @@ -73,7 +73,7 @@ def embrace(s: str): def escape_str(value): if os.name == "nt": - from subprocess import list2cmdline # nosec B404 + from subprocess import list2cmdline return list2cmdline([value]) from shlex import quote diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 670f5423a..f6824e243 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -634,7 +634,7 @@ class Repo: # that just happened to be at the same path as old deleted ones. btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None) - md5 = hashlib.md5( # noqa: S324 # nosec B324, B303 + md5 = hashlib.md5( # noqa: S324 str( ( root_dir, diff --git a/dvc/repo/artifacts.py b/dvc/repo/artifacts.py index a772e6a48..fef2d6f3f 100644 --- a/dvc/repo/artifacts.py +++ b/dvc/repo/artifacts.py @@ -256,7 +256,7 @@ class Artifacts: return len(to_infos), relpath(localfs.path.commonpath(to_infos)) @classmethod - def get( # noqa: C901, PLR0913 + def get( # noqa: PLR0913 cls, url: str, name: str, diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index 9526ba26b..3fb941dfd 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def brancher( # noqa: E302 +def brancher( self, revs=None, all_branches=False, diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index fcf4c2a54..f883eadeb 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -141,7 +141,7 @@ class Experiments: **kwargs, ) - def reproduce_celery( # noqa: C901 + def reproduce_celery( self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs ) -> Dict[str, str]: results: Dict[str, str] = {} diff --git a/dvc/repo/experiments/apply.py b/dvc/repo/experiments/apply.py index 102f2c6c4..11b37c77f 100644 --- a/dvc/repo/experiments/apply.py +++ b/dvc/repo/experiments/apply.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) @locked @scm_context -def apply(repo: "Repo", rev: str, **kwargs): # noqa: C901 +def apply(repo: "Repo", rev: str, **kwargs): from dvc.repo.checkout import checkout as dvc_checkout from dvc.scm import RevError, resolve_rev diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 97a03ce50..c08ba49c0 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -1,6 +1,6 @@ import logging import os -import pickle # nosec B403 +import pickle import shutil from abc import ABC, abstractmethod from contextlib import contextmanager @@ -364,7 +364,7 @@ class BaseExecutor(ABC): @staticmethod def unpack_repro_args(path): with open(path, "rb") as fobj: - data = pickle.load(fobj) # noqa: S301 # nosec B301 + data = pickle.load(fobj) # noqa: S301 return data["args"], data["kwargs"] def fetch_exps( @@ -583,7 +583,7 @@ class BaseExecutor(ABC): @classmethod @contextmanager - def _repro_dvc( # noqa: C901 + def _repro_dvc( cls, info: "ExecutorInfo", infofile: Optional[str] = None, diff --git a/dvc/repo/experiments/queue/base.py b/dvc/repo/experiments/queue/base.py index 863d2bf34..6abdbff8e 100644 --- a/dvc/repo/experiments/queue/base.py +++ b/dvc/repo/experiments/queue/base.py @@ -297,7 +297,7 @@ class BaseStashQueue(ABC): output. """ - def _stash_exp( # noqa: C901 + def _stash_exp( self, *args, params: Optional[Dict[str, List[str]]] = None, @@ -425,7 +425,7 @@ class BaseStashQueue(ABC): return msg def _pack_args(self, *args, **kwargs) -> None: - import pickle # nosec B403 + import pickle if os.path.exists(self.args_file) and self.scm.is_tracked(self.args_file): logger.warning( @@ -440,7 +440,7 @@ class BaseStashQueue(ABC): ) with open(self.args_file, "rb") as fobj: try: - data = pickle.load(fobj) # noqa: S301 # nosec B301 + data = pickle.load(fobj) # noqa: S301 except Exception: # noqa: BLE001, pylint: disable=broad-except data = {} extra = int(data.get("extra", 0)) + 1 diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index bd20dc0e3..76b460813 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -75,7 +75,7 @@ def _collect_indexes( # noqa: PLR0913 @locked -def fetch( # noqa: C901, PLR0913 +def fetch( # noqa: PLR0913 self, targets=None, jobs=None, diff --git a/dvc/repo/imp_url.py b/dvc/repo/imp_url.py index 4deb8ad9c..b243fa811 100644 --- a/dvc/repo/imp_url.py +++ b/dvc/repo/imp_url.py @@ -14,7 +14,7 @@ from . import locked @locked @scm_context -def imp_url( # noqa: C901, PLR0913 +def imp_url( # noqa: PLR0913 self: "Repo", url, out=None, diff --git a/dvc/repo/open_repo.py b/dvc/repo/open_repo.py index 1b5e0bbeb..37a83ae09 100644 --- a/dvc/repo/open_repo.py +++ b/dvc/repo/open_repo.py @@ -150,7 +150,7 @@ def _cached_clone(url, rev): @wrap_with(threading.Lock()) -def _clone_default_branch(url, rev): # noqa: C901, PLR0912 +def _clone_default_branch(url, rev): # noqa: PLR0912 """Get or create a clean clone of the url. The cloned is reactualized with git pull unless rev is a known sha. diff --git a/dvc/repo/push.py b/dvc/repo/push.py index d01638637..e93c52c86 100644 --- a/dvc/repo/push.py +++ b/dvc/repo/push.py @@ -34,7 +34,7 @@ def _update_meta(index, **kwargs): @locked -def push( # noqa: C901, PLR0913 +def push( # noqa: PLR0913 self, targets=None, jobs=None, diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 9a26d07dd..d6aa6e02f 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -559,9 +559,7 @@ class Stage(params.StageParams): raise CacheLinkError(link_failures) @rwlocked(write=["outs"]) - def add_outs( # noqa: C901 - self, filter_info=None, allow_missing: bool = False, **kwargs - ): + def add_outs(self, filter_info=None, allow_missing: bool = False, **kwargs): from dvc.output import OutputDoesNotExistError link_failures = [] @@ -582,7 +580,7 @@ class Stage(params.StageParams): raise CacheLinkError(link_failures) @rwlocked(read=["deps", "outs"]) - def run( # noqa: C901 + def run( self, dry=False, no_commit=False, diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index f2ffa221c..cd124604b 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -165,7 +165,7 @@ class StageLoader(Mapping): return len(self.resolver.get_keys()) def __contains__(self, name): - return self.resolver.has_key(name) # noqa: W601 + return self.resolver.has_key(name) def is_foreach_or_matrix_generated(self, name: str) -> bool: return ( diff --git a/dvc/stage/run.py b/dvc/stage/run.py index 76e3db11d..5892c0d7f 100644 --- a/dvc/stage/run.py +++ b/dvc/stage/run.py @@ -1,7 +1,7 @@ import logging import os import signal -import subprocess # nosec B404 +import subprocess import threading from dvc.utils import fix_env @@ -83,7 +83,7 @@ def _run(executable, cmd, **kwargs): exec_cmd = _make_cmd(executable, cmd) try: - p = subprocess.Popen(exec_cmd, **kwargs) # nosec B603 # noqa: S603 + p = subprocess.Popen(exec_cmd, **kwargs) # noqa: S603 if main_thread: old_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) diff --git a/dvc/testing/benchmarks/cli/stories/test_modify_data.py b/dvc/testing/benchmarks/cli/stories/test_modify_data.py index cb315e613..65a600b41 100644 --- a/dvc/testing/benchmarks/cli/stories/test_modify_data.py +++ b/dvc/testing/benchmarks/cli/stories/test_modify_data.py @@ -16,7 +16,7 @@ def test_partial_add(bench_dvc, tmp_dir, dvc, dataset, remote): # Move some files to create a partial dataset os.makedirs("partial-copy") for f in glob.glob("*", root_dir=dataset, recursive=True): # type: ignore[call-arg] - if random.random() > 0.5: # noqa: S311 # nosec + if random.random() > 0.5: # noqa: S311 shutil.move(dataset / f, tmp_dir / "partial-copy" / f) # Add/push partial dataset @@ -42,7 +42,7 @@ def test_partial_remove(bench_dvc, tmp_dir, dvc, dataset, remote): # Remove some files for f in glob.glob("*", root_dir=dataset, recursive=True): # type: ignore[call-arg] - if random.random() > 0.5: # noqa: S311 # nosec + if random.random() > 0.5: # noqa: S311 if os.path.isfile(dataset / f): os.remove(dataset / f) elif os.path.isdir(dataset / f): diff --git a/dvc/testing/benchmarks/fixtures.py b/dvc/testing/benchmarks/fixtures.py index 0982bb011..18171cac4 100644 --- a/dvc/testing/benchmarks/fixtures.py +++ b/dvc/testing/benchmarks/fixtures.py @@ -1,7 +1,7 @@ import os import shutil from pathlib import Path -from subprocess import check_output # nosec B404 +from subprocess import check_output import pytest from dulwich.porcelain import clone @@ -88,7 +88,7 @@ def make_dvc_bin( dvc_bin = bench_config.dvc_bin def _dvc_bin(*args): - return check_output([dvc_bin, *args], text=True) # nosec B603 # noqa: S603 + return check_output([dvc_bin, *args], text=True) # noqa: S603 _dvc_bin.version = parse_tuple(_dvc_bin("--version")) # type: ignore[attr-defined] return _dvc_bin diff --git a/dvc/testing/fixtures.py b/dvc/testing/fixtures.py index b5b20d702..6767debbd 100644 --- a/dvc/testing/fixtures.py +++ b/dvc/testing/fixtures.py @@ -1,6 +1,6 @@ import os import pathlib -import subprocess # nosec B404 +import subprocess from typing import Dict, Tuple import pytest @@ -236,7 +236,7 @@ def docker_services( pytest.skip("disabled for Windows on CI") try: - subprocess.check_output( # nosec B607, B602, + subprocess.check_output( "docker ps", # noqa: S607 stderr=subprocess.STDOUT, shell=True, # noqa: S602 @@ -250,7 +250,7 @@ def docker_services( subprocess.check_output( cmd, stderr=subprocess.STDOUT, - shell=True, # nosec B602 # noqa: S602 + shell=True, # noqa: S602 ) except subprocess.CalledProcessError as err: out = (err.output or b"").decode("utf-8") diff --git a/dvc/ui/pager.py b/dvc/ui/pager.py index f23153df1..5daebddfb 100644 --- a/dvc/ui/pager.py +++ b/dvc/ui/pager.py @@ -59,7 +59,7 @@ def find_pager(): if not pager: pager = os.getenv(PAGER_ENV) if not pager: - ret = os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") # nosec B605 # noqa: S605 + ret = os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") # noqa: S605 if ret != 0: logger.warning( "Unable to find `less` in the PATH. Check out %s for more info.", diff --git a/pyproject.toml b/pyproject.toml index a3f700f62..dbd209671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -244,8 +244,6 @@ module = [ ignore-words-list = "ba,datas,fo,uptodate,cachable,falsy" [tool.ruff] -# external flake8 codes that should be preserved -external = ["B301", "C901", "E302", "W601"] ignore = [ "N818", "S101", "ISC001", "PT004", "PT007", "RET501", "RET502", "RET503", "SIM105", "SIM108", "SIM117", "TRY003", "TRY200", "TRY300", "PLR2004", "PLW2901", "RUF012" @@ -288,7 +286,3 @@ extend-ignore-names = ["M", "SCM"] [tool.ruff.pylint] max-args = 10 - -[tool.bandit] -exclude_dirs = ["tests", "dvc/repo/experiments/utils.py"] -skips = ["B101"]