diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa176b621..e0b5048d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: sort-simple-yaml - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.1.14' + rev: 'v0.1.15' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/dvc/api/__init__.py b/dvc/api/__init__.py index 82cc13242..5af192dea 100644 --- a/dvc/api/__init__.py +++ b/dvc/api/__init__.py @@ -7,6 +7,7 @@ from .scm import all_branches, all_commits, all_tags from .show import metrics_show, params_show __all__ = [ + "DVCFileSystem", "all_branches", "all_commits", "all_tags", @@ -14,9 +15,8 @@ __all__ = [ "exp_save", "exp_show", "get_url", + "metrics_show", "open", "params_show", - "metrics_show", "read", - "DVCFileSystem", ] diff --git a/dvc/commands/data.py b/dvc/commands/data.py index 82976d527..4e89b6e05 100644 --- a/dvc/commands/data.py +++ b/dvc/commands/data.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar, Dict, Tuple from funcy import chunks, compact, log_durations @@ -17,14 +17,14 @@ logger = logger.getChild(__name__) class CmdDataStatus(CmdBase): - COLORS = { + COLORS: ClassVar[Dict[str, str]] = { "not_in_remote": "red", "not_in_cache": "red", "committed": "green", "uncommitted": "yellow", "untracked": "cyan", } - LABELS = { + LABELS: ClassVar[Dict[str, str]] = { "not_in_remote": "Not in remote", "not_in_cache": "Not in cache", "committed": "DVC committed changes", @@ -32,7 +32,7 @@ class CmdDataStatus(CmdBase): "untracked": "Untracked files", "unchanged": "DVC unchanged files", } - HINTS = { + HINTS: ClassVar[Dict[str, Tuple[str, ...]]] = { "not_in_remote": ('use "dvc push ..." to upload files',), "not_in_cache": ('use "dvc fetch ..." to download files',), "committed": ("git commit the corresponding dvc files to update the repo",), diff --git a/dvc/dependency/db.py b/dvc/dependency/db.py index 10a5dbf29..99ebb0923 100644 --- a/dvc/dependency/db.py +++ b/dvc/dependency/db.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, Optional +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterator, Optional from funcy import compact, log_durations @@ -46,7 +46,7 @@ class DbDependency(AbstractDependency): PARAM_QUERY = "query" PARAM_TABLE = "table" PARAM_FILE_FORMAT = "file_format" - DB_SCHEMA = { + DB_SCHEMA: ClassVar[Dict] = { PARAM_DB: { PARAM_QUERY: str, PARAM_CONNECTION: str, @@ -110,6 +110,7 @@ class DbDependency(AbstractDependency): raise DvcException(f"connection {self.connection} not found in config") file_format = file_format or self.db_info.get(self.PARAM_FILE_FORMAT, "csv") + assert file_format with client(config) as db: msg = "Testing connection" with log_durations(logger.debug, msg), ui.status(msg) as status: diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index d84b4ed9d..42071c5ed 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union import voluptuous as vol @@ -20,7 +20,7 @@ class RepoDependency(Dependency): PARAM_CONFIG = "config" PARAM_REMOTE = "remote" - REPO_SCHEMA = { + REPO_SCHEMA: ClassVar[Dict] = { PARAM_REPO: { vol.Required(PARAM_URL): str, PARAM_REV: str, diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index cad653772..06718ca7f 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -4,6 +4,7 @@ from typing import ( TYPE_CHECKING, Any, Callable, + ClassVar, Dict, List, Optional, @@ -181,10 +182,10 @@ class SingleStageFile(FileMixin): from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA from dvc.stage.loader import SingleStageLoader as LOADER # noqa: N814 - metrics: List[str] = [] - plots: Any = {} - params: List[str] = [] - artifacts: Dict[str, Optional[Dict[str, Any]]] = {} + metrics: ClassVar[List[str]] = [] + plots: ClassVar[Any] = {} + params: ClassVar[List[str]] = [] + artifacts: ClassVar[Dict[str, Optional[Dict[str, Any]]]] = {} @property def stage(self) -> "Stage": diff --git a/dvc/logger.py b/dvc/logger.py index b2fcf8a2d..7256fe5c2 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -5,6 +5,7 @@ import logging.config import logging.handlers import os import sys +from typing import ClassVar, Dict import colorama @@ -78,7 +79,7 @@ class ColorFormatter(logging.Formatter): """ reset = colorama.Fore.RESET - color_codes = { + color_codes: ClassVar[Dict[str, str]] = { "TRACE": colorama.Fore.GREEN, "DEBUG": colorama.Fore.BLUE, "WARNING": colorama.Fore.YELLOW, diff --git a/dvc/parsing/context.py b/dvc/parsing/context.py index 45b5ffdee..7416a156d 100644 --- a/dvc/parsing/context.py +++ b/dvc/parsing/context.py @@ -322,7 +322,7 @@ class Context(CtxDict): continue params_file = self._tracked_data[source] keys = [keys] if isinstance(keys, str) else keys - params_file.update({key: node.value for key in keys}) + params_file.update(dict.fromkeys(keys, node.value)) def select(self, key: str, unwrap: bool = False): """Select the item using key, similar to `__getitem__` diff --git a/dvc/progress.py b/dvc/progress.py index 0793d8367..67d7c8514 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -2,7 +2,7 @@ import logging import sys from threading import RLock -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar, Dict from tqdm import tqdm @@ -33,7 +33,7 @@ class Tqdm(tqdm): " [{elapsed}<{remaining}, {rate_fmt:>11}]" ) BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]" - BYTES_DEFAULTS = { + BYTES_DEFAULTS: ClassVar[Dict[str, Any]] = { "unit": "B", "unit_scale": True, "unit_divisor": 1024, diff --git a/dvc/render/converter/vega.py b/dvc/render/converter/vega.py index b5cb5a3ef..fd97f6c2e 100644 --- a/dvc/render/converter/vega.py +++ b/dvc/render/converter/vega.py @@ -59,7 +59,7 @@ def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str): def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]): - x = properties.get("x", None) + x = properties.get("x") if x is not None and isinstance(x, dict): for filename, field in _file_field(x): _verify_field(file2datapoints, filename, field) @@ -255,7 +255,7 @@ class VegaConverter(Converter): x_file, x_field = xs[i] datapoints = [{**d} for d in file2datapoints.get(y_file, [])] - if props_update.get("y", None) == "dvc_inferred_y_value": + if props_update.get("y") == "dvc_inferred_y_value": _update_from_field( datapoints, field="dvc_inferred_y_value", diff --git a/dvc/repo/experiments/show.py b/dvc/repo/experiments/show.py index 4314a6b15..d80271640 100644 --- a/dvc/repo/experiments/show.py +++ b/dvc/repo/experiments/show.py @@ -130,7 +130,7 @@ def _build_rows( **kwargs, ) -> Iterator[Tuple["CellT", ...]]: for baseline in baseline_states: - row: Dict[str, "CellT"] = {k: fill_value for k in all_headers} + row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value) row["Experiment"] = "" if baseline.name: row["rev"] = baseline.name @@ -245,7 +245,7 @@ def _exp_range_rows( logger.debug("Returning tip commit for legacy checkpoint exp") exp = first(exp_range.revs) if exp: - row: Dict[str, "CellT"] = {k: fill_value for k in all_headers} + row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value) row["Experiment"] = exp.name or "" row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev row["typ"] = "branch_base" if is_base else "branch_commit" diff --git a/dvc/repo/params/show.py b/dvc/repo/params/show.py index 5105a52cc..8873e6032 100644 --- a/dvc/repo/params/show.py +++ b/dvc/repo/params/show.py @@ -76,7 +76,7 @@ def _collect_params( path = fs.from_os_path(param) # make paths absolute for DVCFileSystem repo_path = f"{fs.root_marker}{path}" - ret.update({file: _params for file in expand_paths(fs, [repo_path])}) + ret.update(dict.fromkeys(expand_paths(fs, [repo_path]), _params)) return ret diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index e157d8a20..14e7f9385 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -183,7 +183,7 @@ def _reproduce( to_skip: Dict["Stage", "Stage"] = {} ret: Optional["Stage"] = None - force_state = {node: force for node in stages} + force_state = dict.fromkeys(stages, force) for stage in stages: if stage in to_skip: @@ -203,11 +203,11 @@ def _reproduce( _raise_error(exc, stage) dependents = handle_error(graph, on_error, exc, stage) - to_skip.update({node: stage for node in dependents}) + to_skip.update(dict.fromkeys(dependents, stage)) continue if force_downstream and (ret or force_stage): - force_state.update({node: True for node in downstream}) + force_state.update(dict.fromkeys(downstream, True)) if ret: result.append(ret) diff --git a/dvc/testing/fixtures.py b/dvc/testing/fixtures.py index 6767debbd..551f9e52c 100644 --- a/dvc/testing/fixtures.py +++ b/dvc/testing/fixtures.py @@ -8,29 +8,29 @@ import pytest from .scripts import copy_script __all__ = [ - "make_tmp_dir", - "tmp_dir", - "scm", + "cloud", + "copy_script", + "docker_compose_project_name", + "docker_services", "dvc", + "local_cloud", + "local_remote", + "local_workspace", "make_cloud", "make_cloud_version_aware", "make_local", - "cloud", - "local_cloud", "make_remote", "make_remote_version_aware", "make_remote_worktree", + "make_tmp_dir", + "make_workspace", "remote", "remote_version_aware", "remote_worktree", - "local_remote", - "workspace", - "make_workspace", - "local_workspace", - "docker_compose_project_name", - "docker_services", - "copy_script", "run_copy", + "scm", + "tmp_dir", + "workspace", ] CACHE: Dict[Tuple[bool, bool, bool], str] = {} diff --git a/dvc/testing/path_info.py b/dvc/testing/path_info.py index 48d5a0f0c..820d7f0d2 100644 --- a/dvc/testing/path_info.py +++ b/dvc/testing/path_info.py @@ -1,7 +1,7 @@ import os import pathlib import posixpath -from typing import Callable +from typing import Callable, ClassVar, Dict from urllib.parse import urlparse from dvc.utils import relpath @@ -120,7 +120,12 @@ class _URLPathParents: class URLInfo(_BasePath): - DEFAULT_PORTS = {"http": 80, "https": 443, "ssh": 22, "hdfs": 0} + DEFAULT_PORTS: ClassVar[Dict[str, int]] = { + "http": 80, + "https": 443, + "ssh": 22, + "hdfs": 0, + } def __init__(self, url): p = urlparse(url) diff --git a/pyproject.toml b/pyproject.toml index 3687107ca..3ef3ba2d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -197,7 +197,7 @@ show_error_codes = true show_error_context = true show_traceback = true strict_equality = true -strict_concatenate = true +extra_checks = true warn_no_return = true warn_redundant_casts = true warn_unreachable = true @@ -243,46 +243,50 @@ module = [ ignore-words-list = "ba,datas,fo,uptodate,cachable,falsy" [tool.ruff] +show-source = true +show-fixes = true + +[tool.ruff.lint] ignore = [ "N818", "S101", "ISC001", "PT004", "PT007", "RET501", "RET502", "RET503", "SIM105", "SIM108", - "SIM117", "TRY003", "TRY200", "TRY300", "PLR2004", "PLW2901", "RUF012" + "SIM117", "TRY003", "TRY200", "TRY300", "PLR2004", "PLW2901", ] select = [ "F", "E", "W", "C90", "I", "N", "UP", "YTT", "ASYNC", "S", "BLE", "B", "A", "C4", "T10", "EXE", "ISC", "ICN", "G", "INP", "PIE", "T20", "PYI", "PT", "Q", "RSE", "RET", "SLOT", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY", - "FLY", "PERF101", "RUF", + "FLY", "PERF101", "RUF", "RUF018", "RUF019", "RUF022", "RUF023", "RUF025", ] -show-source = true -show-fixes = true +preview = true +explicit-preview-rules = true -[tool.ruff.flake8-pytest-style] +[tool.ruff.lint.flake8-pytest-style] fixture-parentheses = false mark-parentheses = false parametrize-names-type = "csv" raises-extend-require-match-for = ["dvc.exceptions.DvcException", "dvc.scm.SCMError", "scmrepo.exceptions.SCMError"] -[tool.ruff.flake8-tidy-imports] -[tool.ruff.flake8-tidy-imports.banned-api] +[tool.ruff.lint.flake8-tidy-imports] +[tool.ruff.lint.flake8-tidy-imports.banned-api] "funcy.cached_property" = {msg = "use `from dvc.utils.objects import cached_property` instead."} -[tool.ruff.flake8-type-checking] +[tool.ruff.lint.flake8-type-checking] strict = true -[tool.ruff.flake8-unused-arguments] +[tool.ruff.lint.flake8-unused-arguments] ignore-variadic-names = true -[tool.ruff.isort] +[tool.ruff.lint.isort] known-first-party = ["dvc", "dvc_data", "dvc_objects", "dvc_render", "dvc_task", "tests"] -[tool.ruff.pep8-naming] +[tool.ruff.lint.pep8-naming] extend-ignore-names = ["M", "SCM"] +[tool.ruff.lint.pylint] +max-args = 10 + [tool.ruff.per-file-ignores] "dvc/commands/**" = ["N806"] "dvc/testing/**" = ["ARG002"] "dvc/testing/benchmarks/**" = ["ARG001"] "tests/**" = ["S", "ARG001", "ARG002", "TRY002", "TRY301"] - -[tool.ruff.pylint] -max-args = 10 diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index b340ab89c..3a473de7e 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -49,11 +49,11 @@ import os import pytest __all__ = [ - "run_head", "erepo_dir", "git_dir", - "git_upstream", "git_downstream", + "git_upstream", + "run_head", ] diff --git a/tests/func/repro/test_repro.py b/tests/func/repro/test_repro.py index 34f6820b5..044130485 100644 --- a/tests/func/repro/test_repro.py +++ b/tests/func/repro/test_repro.py @@ -473,7 +473,7 @@ def test_repro_allow_missing(tmp_dir, dvc): tmp_dir.gen("fixed", "fixed") dvc.stage.add(name="create-foo", cmd="echo foo > foo", deps=["fixed"], outs=["foo"]) dvc.stage.add(name="copy-foo", cmd="cp foo bar", deps=["foo"], outs=["bar"]) - (create_foo, copy_foo) = dvc.reproduce() + (create_foo, _) = dvc.reproduce() remove("foo") remove(create_foo.outs[0].cache_path) diff --git a/tests/func/repro/test_repro_allow_missing.py b/tests/func/repro/test_repro_allow_missing.py index 26041781c..e69b3764e 100644 --- a/tests/func/repro/test_repro_allow_missing.py +++ b/tests/func/repro/test_repro_allow_missing.py @@ -5,7 +5,7 @@ def test_repro_allow_missing(tmp_dir, dvc): tmp_dir.gen("fixed", "fixed") dvc.stage.add(name="create-foo", cmd="echo foo > foo", deps=["fixed"], outs=["foo"]) dvc.stage.add(name="copy-foo", cmd="cp foo bar", deps=["foo"], outs=["bar"]) - (create_foo, copy_foo) = dvc.reproduce() + (create_foo, _) = dvc.reproduce() remove("foo") remove(create_foo.outs[0].cache_path) diff --git a/tests/func/test_daemon.py b/tests/func/test_daemon.py index 6d38e5a19..291aca04e 100644 --- a/tests/func/test_daemon.py +++ b/tests/func/test_daemon.py @@ -8,7 +8,7 @@ from contextlib import contextmanager, suppress from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path from threading import Thread -from typing import Dict, Iterator +from typing import ClassVar, Dict, Iterator import psutil import pytest @@ -42,7 +42,7 @@ UPDATER_INFO_STR = json.dumps(UPDATER_INFO).encode("utf8") def make_request_handler(): class RequestHandler(SimpleHTTPRequestHandler): # save requests count for each method - hits: Dict[str, int] = defaultdict(int) + hits: ClassVar[Dict[str, int]] = defaultdict(int) def log_message(self, format, *args) -> None: # noqa: A002 super().log_message(format, *args) diff --git a/tests/func/test_dvcfile.py b/tests/func/test_dvcfile.py index c433d5a50..1950114e2 100644 --- a/tests/func/test_dvcfile.py +++ b/tests/func/test_dvcfile.py @@ -169,7 +169,7 @@ def test_remove_stage(tmp_dir, dvc, run_copy): dvc_file.remove_stage(stage) - assert ["copy-bar-foobar"] == list(dvc_file._load()[0]["stages"].keys()) + assert list(dvc_file._load()[0]["stages"].keys()) == ["copy-bar-foobar"] # sanity check stage2.reload() @@ -193,7 +193,7 @@ def test_remove_stage_lockfile(tmp_dir, dvc, run_copy): assert {"copy-bar-foobar", "copy-foo-bar"} == set(lock_file.load()["stages"].keys()) lock_file.remove_stage(stage) - assert ["copy-bar-foobar"] == list(lock_file.load()["stages"].keys()) + assert list(lock_file.load()["stages"].keys()) == ["copy-bar-foobar"] # sanity check stage2.reload() diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index af42c0a1a..5df851e8a 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -35,14 +35,14 @@ def good_and_bad_cache(tmp_dir, dvc): def test_gc_api(dvc, good_and_bad_cache): dvc.gc(workspace=True) odb = dvc.cache.local - good_cache, bad_cache = good_and_bad_cache + good_cache, _ = good_and_bad_cache assert set(odb.all()) == good_cache def test_gc_cli(dvc, good_and_bad_cache): assert main(["gc", "-wf"]) == 0 odb = dvc.cache.local - good_cache, bad_cache = good_and_bad_cache + good_cache, _ = good_and_bad_cache assert set(odb.all()) == good_cache @@ -113,7 +113,7 @@ def test_gc_no_dir_cache(tmp_dir, dvc): with pytest.raises(RevCollectionError) as exc: dvc.gc(workspace=True) - assert type(exc.value.__cause__) == CollectCacheError + assert type(exc.value.__cause__) is CollectCacheError assert _count_files(dvc.cache.local.path) == 4 dvc.gc(force=True, workspace=True) @@ -319,7 +319,7 @@ def test_date(tmp_dir, scm, dvc): def test_gc_not_in_remote(tmp_dir, scm, dvc, mocker, local_remote): - (standalone, dir1, dir2) = tmp_dir.dvc_gen( + (standalone, dir1, _) = tmp_dir.dvc_gen( { "file1": "standalone", "dir1": {"file2": "file2"}, @@ -429,7 +429,7 @@ def test_gc_cloud_remote_field(tmp_dir, scm, dvc, mocker, make_remote): def test_gc_dry(dvc, good_and_bad_cache): dvc.gc(workspace=True, dry=True) odb = dvc.cache.local - good_cache, bad_cache = good_and_bad_cache + good_cache, _ = good_and_bad_cache assert set(odb.all()) != good_cache diff --git a/tests/unit/output/test_output.py b/tests/unit/output/test_output.py index d734f3bbf..f05b0b3ef 100644 --- a/tests/unit/output/test_output.py +++ b/tests/unit/output/test_output.py @@ -102,7 +102,7 @@ def test_get_used_objs(exists, expected_message, mocker, caplog): ).return_value = exists with caplog.at_level(logging.WARNING, logger="dvc"): - assert {} == output.get_used_objs() + assert output.get_used_objs() == {} assert first(caplog.messages) == expected_message diff --git a/tests/unit/repo/experiments/test_collect.py b/tests/unit/repo/experiments/test_collect.py index 478809f94..e1342ed2c 100644 --- a/tests/unit/repo/experiments/test_collect.py +++ b/tests/unit/repo/experiments/test_collect.py @@ -42,7 +42,7 @@ def test_collect_stable_sorting(dvc, scm, mocker): # shuffle collection order random.shuffle(exp_ranges) - return {baseline_rev: exp_ranges for baseline_rev in baseline_revs} + return dict.fromkeys(baseline_revs, exp_ranges) mocker.patch("dvc.repo.experiments.collect.collect_queued", collect_queued_patched) mocker.patch("dvc.repo.experiments.collect.collect_active", return_value={}) diff --git a/tests/unit/test_lockfile.py b/tests/unit/test_lockfile.py index 73e6fe19b..b1bbd10f7 100644 --- a/tests/unit/test_lockfile.py +++ b/tests/unit/test_lockfile.py @@ -58,7 +58,7 @@ def test_stage_overwrites_if_already_exists(tmp_dir, dvc): def test_load_when_lockfile_does_not_exist(tmp_dir, dvc): - assert {} == Lockfile(dvc, "pipelines.lock").load() + assert Lockfile(dvc, "pipelines.lock").load() == {} @pytest.mark.parametrize( diff --git a/tests/utils/asserts.py b/tests/utils/asserts.py index ae589fe93..301c7422a 100644 --- a/tests/utils/asserts.py +++ b/tests/utils/asserts.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: def issubset(subset: Dict, superset: Dict) -> bool: - assert {**superset, **subset} == superset + assert superset == {**superset, **subset} return True