From 90a27179eb0841ed5af366f7f09a132cfbe59547 Mon Sep 17 00:00:00 2001 From: skshetry <18718008+skshetry@users.noreply.github.com> Date: Wed, 11 Jan 2023 17:58:51 +0545 Subject: [PATCH] ruff: ignore complex functions and remove dvc.utils.fs.makedirs (#8795) We may want to prevent any new functions from getting complex. --- .flake8 | 2 +- dvc/commands/data.py | 2 +- dvc/commands/experiments/show.py | 4 +-- dvc/commands/gc.py | 2 +- dvc/commands/plots.py | 2 +- dvc/compare.py | 4 +-- dvc/dagascii.py | 2 +- dvc/external_repo.py | 2 +- dvc/fs/dvc.py | 4 ++- dvc/machine/backend/terraform.py | 5 ++-- dvc/output.py | 4 +-- dvc/parsing/interpolate.py | 2 +- dvc/proc/process.py | 4 +-- dvc/render/converter/vega.py | 2 +- dvc/repo/diff.py | 2 +- dvc/repo/experiments/__init__.py | 2 +- dvc/repo/experiments/apply.py | 2 +- dvc/repo/experiments/executor/base.py | 8 +++--- dvc/repo/experiments/executor/local.py | 4 +-- dvc/repo/experiments/pull.py | 2 +- dvc/repo/experiments/push.py | 2 +- dvc/repo/experiments/queue/remove.py | 2 +- dvc/repo/experiments/remove.py | 2 +- dvc/repo/experiments/run.py | 2 +- dvc/repo/experiments/show.py | 2 +- dvc/repo/fetch.py | 2 +- dvc/repo/imp_url.py | 2 +- dvc/repo/imports.py | 4 +-- dvc/repo/push.py | 2 +- dvc/repo/reproduce.py | 4 +-- dvc/repo/stage.py | 2 +- dvc/repo/update.py | 2 +- dvc/rwlock.py | 2 +- dvc/scm.py | 2 +- dvc/testing/tmp_dir.py | 5 ++-- dvc/utils/collections.py | 2 +- dvc/utils/fs.py | 35 -------------------------- dvc/utils/strictyaml.py | 2 +- pyproject.toml | 6 +---- tests/func/experiments/test_show.py | 9 +++---- tests/func/test_external_repo.py | 4 +-- tests/func/test_get.py | 3 +-- tests/func/test_import.py | 4 +-- tests/func/test_import_url.py | 3 +-- tests/func/utils/test_fs.py | 20 --------------- tests/unit/utils/test_fs.py | 3 +-- 46 files changed, 61 insertions(+), 128 deletions(-) delete mode 100644 tests/func/utils/test_fs.py diff --git a/.flake8 b/.flake8 index bf51a3627..4060ebaaa 100644 --- a/.flake8 +++ b/.flake8 @@ -6,7 +6,7 @@ ignore= P1, # unindexed parameters in the str.format, see: # https://pypi.org/project/flake8-string-format/ max_line_length = 79 -max-complexity = 15 +max-complexity = 10 select = B,C,E,F,W,T4,B902,T,P show_source = true count = true diff --git a/dvc/commands/data.py b/dvc/commands/data.py index 5f7279270..eca85b575 100644 --- a/dvc/commands/data.py +++ b/dvc/commands/data.py @@ -72,7 +72,7 @@ class CmdDataStatus(CmdBase): yield stage, items @classmethod - def _show_status(cls, status: "DataStatus") -> int: + def _show_status(cls, status: "DataStatus") -> int: # noqa: C901 git_info = status.pop("git") # type: ignore[misc] result = dict(cls._process_status(status)) if not result: diff --git a/dvc/commands/experiments/show.py b/dvc/commands/experiments/show.py index efd0d5be3..208d3a347 100644 --- a/dvc/commands/experiments/show.py +++ b/dvc/commands/experiments/show.py @@ -62,7 +62,7 @@ experiment_types = { } -def _collect_rows( +def _collect_rows( # noqa: C901 base_rev, experiments, all_headers, @@ -314,7 +314,7 @@ def baseline_styler(typ): return {"style": "bold"} if typ == "baseline" else {} -def show_experiments( +def show_experiments( # noqa: C901 all_experiments, keep=None, drop=None, diff --git a/dvc/commands/gc.py b/dvc/commands/gc.py index e29c95096..588b36016 100644 --- a/dvc/commands/gc.py +++ b/dvc/commands/gc.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) class CmdGC(CmdBase): - def run(self): + def run(self): # noqa: C901 from dvc.repo.gc import _validate_args _validate_args( diff --git a/dvc/commands/plots.py b/dvc/commands/plots.py index ddca2147d..dbf6e2ddd 100644 --- a/dvc/commands/plots.py +++ b/dvc/commands/plots.py @@ -94,7 +94,7 @@ class CmdPlots(CmdBase): ) return html_template_path - def run(self): + def run(self): # noqa: C901 from pathlib import Path from dvc.render.match import match_defs_renderers diff --git a/dvc/compare.py b/dvc/compare.py index f1f5fd803..6f6c77a81 100644 --- a/dvc/compare.py +++ b/dvc/compare.py @@ -216,7 +216,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): {k: self._columns[k][i] for k in keys} for i in range(len(self)) ] - def dropna( + def dropna( # noqa: C901 self, axis: str = "rows", how="any", @@ -267,7 +267,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): else: self.drop(*to_drop) - def drop_duplicates( + def drop_duplicates( # noqa: C901 self, axis: str = "rows", subset: Optional[Iterable[str]] = None, diff --git a/dvc/dagascii.py b/dvc/dagascii.py index dfe5c7721..c4c25b8a2 100644 --- a/dvc/dagascii.py +++ b/dvc/dagascii.py @@ -80,7 +80,7 @@ class AsciiCanvas: self.canvas[y][x] = char - def line(self, x0, y0, x1, y1, char): + def line(self, x0, y0, x1, y1, char): # noqa: C901 """Create a line on ASCII canvas. Args: diff --git a/dvc/external_repo.py b/dvc/external_repo.py index fb7c4619f..c7a3eb4c4 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -185,7 +185,7 @@ def _cached_clone(url, rev, for_write=False): @wrap_with(threading.Lock()) -def _clone_default_branch(url, rev, for_write=False): +def _clone_default_branch(url, rev, for_write=False): # noqa: C901 """Get or create a clean clone of the url. The cloned is reactualized with git pull unless rev is a known sha. diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index bc1893384..c373bd769 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -321,7 +321,9 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method ignore_subrepos = kwargs.get("ignore_subrepos", True) return self._info(key, path, ignore_subrepos=ignore_subrepos) - def _info(self, key, path, ignore_subrepos=True, check_ignored=True): + def _info( # noqa: C901 + self, key, path, ignore_subrepos=True, check_ignored=True + ): repo, dvc_fs, subkey = self._get_subrepo_info(key) dvc_info = None diff --git a/dvc/machine/backend/terraform.py b/dvc/machine/backend/terraform.py index 6922b5efa..953986fcd 100644 --- a/dvc/machine/backend/terraform.py +++ b/dvc/machine/backend/terraform.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Iterator, Optional from dvc_ssh import DEFAULT_PORT, SSHFileSystem from dvc.exceptions import DvcException -from dvc.utils.fs import makedirs from .base import BaseMachineBackend @@ -30,7 +29,7 @@ def _sshfs(resource: dict): class TerraformBackend(BaseMachineBackend): def __init__(self, tmp_dir: "StrPath", **kwargs): super().__init__(tmp_dir, **kwargs) - makedirs(tmp_dir, exist_ok=True) + os.makedirs(tmp_dir, exist_ok=True) @contextmanager def make_tpi(self, name: str): @@ -39,7 +38,7 @@ class TerraformBackend(BaseMachineBackend): try: working_dir = os.path.join(self.tmp_dir, name) - makedirs(working_dir, exist_ok=True) + os.makedirs(working_dir, exist_ok=True) yield TPIBackend(working_dir=working_dir) except TPIError as exc: raise DvcException("TPI operation failed") from exc diff --git a/dvc/output.py b/dvc/output.py index a4b209901..25c052e33 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -742,7 +742,7 @@ class Output: ) return checkout_obj - def dumpd(self, **kwargs): + def dumpd(self, **kwargs): # noqa: C901 meta = self.meta.to_dict() meta.pop("isdir", None) ret = {**self.hash_info.to_dict(), **meta} @@ -1034,7 +1034,7 @@ class Output: return obj.filter(prefix) return obj - def get_used_objs( + def get_used_objs( # noqa: C901 self, **kwargs ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]: """Return filtered set of used object IDs for this out.""" diff --git a/dvc/parsing/interpolate.py b/dvc/parsing/interpolate.py index 93e44be4d..d31931e1b 100644 --- a/dvc/parsing/interpolate.py +++ b/dvc/parsing/interpolate.py @@ -92,7 +92,7 @@ def _(obj: bool): @to_str.register(dict) -def _(obj: dict): +def _(obj: dict): # noqa: C901 from dvc.config import Config config = Config().get("parsing", {}) diff --git a/dvc/proc/process.py b/dvc/proc/process.py index edbce6526..6fe1d680b 100644 --- a/dvc/proc/process.py +++ b/dvc/proc/process.py @@ -10,8 +10,6 @@ from typing import List, Optional, TextIO, Union from funcy import cached_property from shortuuid import uuid -from dvc.utils.fs import makedirs - from .exceptions import TimeoutExpired logger = logging.getLogger(__name__) @@ -103,7 +101,7 @@ class ManagedProcess(AbstractContextManager): def _make_wdir(self): if self.wdir: - makedirs(self.wdir, exist_ok=True) + os.makedirs(self.wdir, exist_ok=True) def _dump(self): self._make_wdir() diff --git a/dvc/render/converter/vega.py b/dvc/render/converter/vega.py index 74362c958..879fc5e19 100644 --- a/dvc/render/converter/vega.py +++ b/dvc/render/converter/vega.py @@ -191,7 +191,7 @@ class VegaConverter(Converter): return first(fields) return "x" - def flat_datapoints(self, revision): + def flat_datapoints(self, revision): # noqa: C901 file2datapoints, properties = self.convert() diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index f125d7a7a..f5d330cc5 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -113,7 +113,7 @@ def _paths_checksums(repo, targets): return dict(_output_paths(repo, targets)) -def _output_paths(repo, targets): +def _output_paths(repo, targets): # noqa: C901 from dvc.fs import LocalFileSystem from dvc_data.hashfile.build import build diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index bcf888bdf..d9a5cdc09 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -160,7 +160,7 @@ class Experiments: return last_applied return None - def reproduce_celery( + def reproduce_celery( # noqa: C901 self, entries: Optional[Iterable[QueueEntry]] = None, **kwargs ) -> Dict[str, str]: results: Dict[str, str] = {} diff --git a/dvc/repo/experiments/apply.py b/dvc/repo/experiments/apply.py index 3fe2fce2b..44df36e4c 100644 --- a/dvc/repo/experiments/apply.py +++ b/dvc/repo/experiments/apply.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) @locked @scm_context -def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): +def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901 from scmrepo.exceptions import SCMError as _SCMError from dvc.repo.checkout import checkout as dvc_checkout diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index ff8e9ae12..2ddf74d24 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -102,10 +102,9 @@ class ExecutorInfo: ) def dump_json(self, filename: str): - from dvc.utils.fs import makedirs from dvc.utils.serialize import modify_json - makedirs(os.path.dirname(filename), exist_ok=True) + os.makedirs(os.path.dirname(filename), exist_ok=True) with modify_json(filename) as d: d.update(self.asdict()) @@ -288,10 +287,9 @@ class BaseExecutor(ABC): open_func = fs.open fs.makedirs(dpath) else: - from dvc.utils.fs import makedirs open_func = open - makedirs(dpath, exist_ok=True) + os.makedirs(dpath, exist_ok=True) data = {"args": args, "kwargs": kwargs} if extra is not None: @@ -549,7 +547,7 @@ class BaseExecutor(ABC): @classmethod @contextmanager - def _repro_dvc( + def _repro_dvc( # noqa: C901 cls, info: "ExecutorInfo", infofile: str = None, diff --git a/dvc/repo/experiments/executor/local.py b/dvc/repo/experiments/executor/local.py index 22e4148de..45de0da9e 100644 --- a/dvc/repo/experiments/executor/local.py +++ b/dvc/repo/experiments/executor/local.py @@ -22,7 +22,7 @@ from dvc.repo.experiments.refs import ( ) from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock from dvc.scm import SCM, GitMergeError -from dvc.utils.fs import makedirs, remove +from dvc.utils.fs import remove from .base import BaseExecutor, TaskStatus @@ -166,7 +166,7 @@ class TempDirExecutor(BaseLocalExecutor): **kwargs, ): parent_dir: str = wdir or os.path.join(repo.tmp_dir, EXEC_TMP_DIR) - makedirs(parent_dir, exist_ok=True) + os.makedirs(parent_dir, exist_ok=True) tmp_dir = mkdtemp(dir=parent_dir) try: executor = cls._from_stash_entry(repo, entry, tmp_dir, **kwargs) diff --git a/dvc/repo/experiments/pull.py b/dvc/repo/experiments/pull.py index 3bed3aa87..42de96f1e 100644 --- a/dvc/repo/experiments/pull.py +++ b/dvc/repo/experiments/pull.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) @locked @scm_context -def pull( +def pull( # noqa: C901 repo, git_remote: str, exp_names: Union[Iterable[str], str], diff --git a/dvc/repo/experiments/push.py b/dvc/repo/experiments/push.py index 73fa54e1d..62eaf3c72 100644 --- a/dvc/repo/experiments/push.py +++ b/dvc/repo/experiments/push.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) @locked @scm_context -def push( +def push( # noqa: C901 repo, git_remote: str, exp_names: Union[Iterable[str], str], diff --git a/dvc/repo/experiments/queue/remove.py b/dvc/repo/experiments/queue/remove.py index 605e5effa..3d642a63e 100644 --- a/dvc/repo/experiments/queue/remove.py +++ b/dvc/repo/experiments/queue/remove.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from dvc.repo.experiments.stash import ExpStashEntry -def remove_tasks( +def remove_tasks( # noqa: C901 celery_queue: "LocalCeleryQueue", queue_entries: Iterable["QueueEntry"], ): diff --git a/dvc/repo/experiments/remove.py b/dvc/repo/experiments/remove.py index b2666fbbf..d0a9148db 100644 --- a/dvc/repo/experiments/remove.py +++ b/dvc/repo/experiments/remove.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) @locked @scm_context -def remove( +def remove( # noqa: C901 repo: "Repo", exp_names: Union[None, str, List[str]] = None, rev: Optional[str] = None, diff --git a/dvc/repo/experiments/run.py b/dvc/repo/experiments/run.py index 06ae7d333..10836faed 100644 --- a/dvc/repo/experiments/run.py +++ b/dvc/repo/experiments/run.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) @locked -def run( +def run( # noqa: C901 repo, targets: Optional[Iterable[str]] = None, params: Optional[Iterable[str]] = None, diff --git a/dvc/repo/experiments/show.py b/dvc/repo/experiments/show.py index 682332aa5..0fb88f6aa 100644 --- a/dvc/repo/experiments/show.py +++ b/dvc/repo/experiments/show.py @@ -237,7 +237,7 @@ def get_branch_names( return names -def update_names( +def update_names( # noqa: C901 repo: "Repo", branch_names: Dict[str, Optional[str]], result: Dict[str, Dict[str, Any]], diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 997068322..7f9718d1f 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) @locked -def fetch( +def fetch( # noqa: C901 self, targets=None, jobs=None, diff --git a/dvc/repo/imp_url.py b/dvc/repo/imp_url.py index 23a84dd5e..eeb0abb18 100644 --- a/dvc/repo/imp_url.py +++ b/dvc/repo/imp_url.py @@ -10,7 +10,7 @@ from . import locked @locked @scm_context -def imp_url( +def imp_url( # noqa: C901 self, url, out=None, diff --git a/dvc/repo/imports.py b/dvc/repo/imports.py index a62072edf..4925ba56f 100644 --- a/dvc/repo/imports.py +++ b/dvc/repo/imports.py @@ -1,4 +1,5 @@ import logging +import os from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, List, Set, Tuple, Union @@ -94,7 +95,6 @@ def save_imports( Objects which were downloaded from source location. """ from dvc.stage.exceptions import DataSourceChanged - from dvc.utils.fs import makedirs from dvc_data.index import checkout, md5, save from dvc_objects.fs.callbacks import Callback @@ -110,7 +110,7 @@ def save_imports( if len(data_view): cache = repo.odb.local if not cache.fs.exists(cache.path): - makedirs(cache.path) + os.makedirs(cache.path) with TemporaryDirectory(dir=cache.path) as tmpdir: with Callback.as_tqdm_callback( desc="Downloading imports from source", diff --git a/dvc/repo/push.py b/dvc/repo/push.py index 9bce248a8..cc4d7896a 100644 --- a/dvc/repo/push.py +++ b/dvc/repo/push.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: @locked -def push( +def push( # noqa: C901 self, targets=None, jobs=None, diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 7f5de45e7..16545cd81 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -86,7 +86,7 @@ def _track_stage(stage: "Stage") -> None: @locked @scm_context -def reproduce( +def reproduce( # noqa: C901 self: "Repo", targets=None, recursive=False, @@ -138,7 +138,7 @@ def reproduce( return _reproduce_stages(self.index.graph, list(stages), **kwargs) -def _reproduce_stages( +def _reproduce_stages( # noqa: C901 G, stages, downstream=False, single_item=False, on_unchanged=None, **kwargs ): r"""Derive the evaluation of the given node for the given graph. diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index 61fde9c59..ec6702994 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -527,7 +527,7 @@ class StageLoad: stages_ = [stages[key] for key in keys] return stages_, dvcfile - def _collect_all_from_repo( + def _collect_all_from_repo( # noqa: C901 self, onerror: Callable[[str, Exception], None] = None ): """Collects all of the stages present in the DVC repo. diff --git a/dvc/repo/update.py b/dvc/repo/update.py index dc016d50a..19a4df8df 100644 --- a/dvc/repo/update.py +++ b/dvc/repo/update.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: @locked -def update( +def update( # noqa: C901 self, targets=None, rev=None, diff --git a/dvc/rwlock.py b/dvc/rwlock.py index f186bf2a0..f8e1afe91 100644 --- a/dvc/rwlock.py +++ b/dvc/rwlock.py @@ -73,7 +73,7 @@ def _infos_to_str(infos): ) -def _check_blockers(tmp_dir, lock, info, *, mode, waiters): +def _check_blockers(tmp_dir, lock, info, *, mode, waiters): # noqa: C901 from .lock import LockError non_existing_pid = set() diff --git a/dvc/scm.py b/dvc/scm.py index 0d32c1055..8d0ad8b3b 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -195,7 +195,7 @@ def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]: return results -def iter_revs( +def iter_revs( # noqa: C901 scm: "Git", revs: Optional[List[str]] = None, num: int = 1, diff --git a/dvc/testing/tmp_dir.py b/dvc/testing/tmp_dir.py index ea4cd0d77..eca2a19e0 100644 --- a/dvc/testing/tmp_dir.py +++ b/dvc/testing/tmp_dir.py @@ -52,7 +52,6 @@ from contextlib import contextmanager from functools import partialmethod from dvc.utils import serialize -from dvc.utils.fs import makedirs class TmpDir(pathlib.Path): @@ -138,11 +137,11 @@ class TmpDir(pathlib.Path): if isinstance(contents, dict): if not contents: - makedirs(path, exist_ok=True) + os.makedirs(path, exist_ok=True) else: self._gen(contents, prefix=path) else: - makedirs(path.parent, exist_ok=True) + os.makedirs(path.parent, exist_ok=True) if isinstance(contents, bytes): path.write_bytes(contents) else: diff --git a/dvc/utils/collections.py b/dvc/utils/collections.py index dd04202d7..f22af3b55 100644 --- a/dvc/utils/collections.py +++ b/dvc/utils/collections.py @@ -4,7 +4,7 @@ from functools import wraps from typing import Callable, Dict, Iterable, List, TypeVar, Union -def apply_diff(src, dest): +def apply_diff(src, dest): # noqa: C901 """Recursively apply changes from src to dest. Preserves dest type and hidden info in dest structure, diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index 1376dd065..2dd89a291 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -83,38 +83,3 @@ def path_isin(child: "StrPath", parent: "StrPath") -> bool: parent = os.path.join(normalize_path(parent), "") child = normalize_path(child) return child != parent and child.startswith(parent) - - -def makedirs(path, exist_ok=False, mode=None): - if mode is None: - os.makedirs(path, exist_ok=exist_ok) - return - - # Modified version of os.makedirs() with support for extended mode - # (e.g. S_ISGID) - head, tail = os.path.split(path) - if not tail: - head, tail = os.path.split(head) - if head and tail and not os.path.exists(head): - try: - makedirs(head, exist_ok=exist_ok, mode=mode) - except FileExistsError: - # Defeats race condition when another thread created the path - pass - cdir = os.curdir - if isinstance(tail, bytes): - cdir = bytes(os.curdir, "ASCII") - if tail == cdir: # xxx/newdir/. exists if xxx/newdir exists - return - try: - os.mkdir(path, mode) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not os.path.isdir(path): - raise - - try: - os.chmod(path, mode) - except OSError: - logger.trace("failed to chmod '%o' '%s'", mode, path, exc_info=True) diff --git a/dvc/utils/strictyaml.py b/dvc/utils/strictyaml.py index 59d1e1152..02176605c 100644 --- a/dvc/utils/strictyaml.py +++ b/dvc/utils/strictyaml.py @@ -87,7 +87,7 @@ class YAMLSyntaxError(PrettyDvcException, YAMLFileCorruptedError): self.rev: Optional[str] = rev super().__init__(self.path) - def __pretty_exc__(self, **kwargs: Any) -> None: + def __pretty_exc__(self, **kwargs: Any) -> None: # noqa: C901 from ruamel.yaml.error import MarkedYAMLError exc = self.exc.__cause__ diff --git a/pyproject.toml b/pyproject.toml index 4a432ab5f..21562249d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,19 +162,15 @@ line_length = 79 [tool.ruff] line-length = 79 target-version = "py38" -format = "grouped" show-source = true ignore = ["S101", "SIM105", "SIM108", "SIM110", "SIM111", "SIM117", "A003", "ARG002"] select = ["F", "E", "W", "C90", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "ISC", "ICN", "T20", "PT003", "PT008", "PT009", "PT010", "PT015", "PT016", "PT017", "PT020", "PT021", "PT024", "PT025", "PT026", "Q", "RET501", "RET504", "SIM", "ARG", "TID", "PGH", "PLC", "PLE", "PLR", "PLW", "PIE", "RUF"] # external flake8 codes that should be preserved -external = ["B301", "B024", "C901", "E302", "C901", "F401", "W601"] +external = ["B301", "B024", "C901", "E302", "F401", "W601"] [tool.ruff.per-file-ignores] "scripts/**/*.py" = ["T201"] -[tool.ruff.mccabe] -max-complexity = 15 - [tool.ruff.flake8-pytest-style] fixture-parentheses = false mark-parentheses = false diff --git a/tests/func/experiments/test_show.py b/tests/func/experiments/test_show.py index 1ba3655e0..cec63742b 100644 --- a/tests/func/experiments/test_show.py +++ b/tests/func/experiments/test_show.py @@ -22,7 +22,6 @@ from dvc.repo.experiments.utils import ( exp_refs_by_rev, ) from dvc.utils import relpath -from dvc.utils.fs import makedirs from dvc.utils.serialize import YAMLFileCorruptedError LOCK_CONTENTS = { @@ -447,7 +446,7 @@ def test_show_running_workspace( "workspace", f"workspace{BaseExecutor.INFOFILE_EXT}", ) - makedirs(os.path.dirname(pidfile), True) + os.makedirs(os.path.dirname(pidfile), exist_ok=True) (tmp_dir / pidfile).dump_json(info.asdict()) (tmp_dir / lock_file).dump_json(LOCK_CONTENTS) @@ -524,7 +523,7 @@ def test_show_running_tempdir(tmp_dir, scm, dvc, exp_stage, mocker): ) info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION) pidfile = queue.get_infofile_path(stash_rev) - makedirs(os.path.dirname(pidfile), True) + os.makedirs(os.path.dirname(pidfile), exist_ok=True) (tmp_dir / pidfile).dump_json(info.asdict()) mock_fetch = mocker.patch.object( dvc.experiments.tempdir_queue, @@ -557,7 +556,7 @@ def test_show_running_celery(tmp_dir, scm, dvc, exp_stage, mocker): ) info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION) pidfile = queue.get_infofile_path(entries[0].stash_rev) - makedirs(os.path.dirname(pidfile), True) + os.makedirs(os.path.dirname(pidfile), exist_ok=True) (tmp_dir / pidfile).dump_json(info.asdict()) results = dvc.experiments.show() @@ -599,7 +598,7 @@ def test_show_running_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, mocker): location=TempDirExecutor.DEFAULT_LOCATION, status=TaskStatus.RUNNING, ) - makedirs(os.path.dirname(pidfile), True) + os.makedirs(os.path.dirname(pidfile), exist_ok=True) (tmp_dir / pidfile).dump_json(info.asdict()) mocker.patch.object( diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index e6a179b77..9528e9929 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -6,7 +6,7 @@ from scmrepo.git import Git from dvc.external_repo import CLONES, external_repo from dvc.testing.tmp_dir import make_subrepo from dvc.utils import relpath -from dvc.utils.fs import makedirs, remove +from dvc.utils.fs import remove from dvc_data.hashfile.build import build from dvc_data.hashfile.transfer import transfer @@ -200,7 +200,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): # clear cache to test saving to cache cache_dir = tmp_dir / repo.odb.local.path remove(cache_dir) - makedirs(cache_dir) + os.makedirs(cache_dir) staging, _, obj = build( repo.odb.local, diff --git a/tests/func/test_get.py b/tests/func/test_get.py index d2d7eb3cf..0ce2259ae 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -9,7 +9,6 @@ from dvc.odbmgr import ODBManager from dvc.repo import Repo from dvc.repo.get import GetDVCFileError from dvc.testing.tmp_dir import make_subrepo -from dvc.utils.fs import makedirs def test_get_repo_file(tmp_dir, erepo_dir): @@ -170,7 +169,7 @@ def test_get_to_dir(tmp_dir, erepo_dir, dname): with erepo_dir.chdir(): erepo_dir.dvc_gen("file", "contents", commit="create file") - makedirs(dname, exist_ok=True) + os.makedirs(dname, exist_ok=True) Repo.get(os.fspath(erepo_dir), "file", dname) diff --git a/tests/func/test_import.py b/tests/func/test_import.py index b687c6656..9f51b1a15 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -13,7 +13,7 @@ from dvc.fs import system from dvc.odbmgr import ODBManager from dvc.stage.exceptions import StagePathNotFoundError from dvc.testing.tmp_dir import make_subrepo -from dvc.utils.fs import makedirs, remove +from dvc.utils.fs import remove def test_import(tmp_dir, scm, dvc, erepo_dir): @@ -376,7 +376,7 @@ def test_download_error_pulling_imported_stage( @pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"]) def test_import_to_dir(dname, tmp_dir, dvc, erepo_dir): - makedirs(dname, exist_ok=True) + os.makedirs(dname, exist_ok=True) with erepo_dir.chdir(): erepo_dir.dvc_gen("foo", "foo content", commit="create foo") diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index 55cdf9f5f..05bde6a8c 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -12,7 +12,6 @@ from dvc.dvcfile import Dvcfile from dvc.exceptions import InvalidArgumentError from dvc.stage import Stage from dvc.testing.workspace_tests import TestImport as _TestImport -from dvc.utils.fs import makedirs def test_cmd_import(tmp_dir, dvc): @@ -74,7 +73,7 @@ def test_import_url_to_dir(dname, tmp_dir, dvc): tmp_dir.gen({"data_dir": {"file": "file content"}}) src = os.path.join("data_dir", "file") - makedirs(dname, exist_ok=True) + os.makedirs(dname, exist_ok=True) stage = dvc.imp_url(src, dname) diff --git a/tests/func/utils/test_fs.py b/tests/func/utils/test_fs.py deleted file mode 100644 index be5dcb3ac..000000000 --- a/tests/func/utils/test_fs.py +++ /dev/null @@ -1,20 +0,0 @@ -import os -import stat - -import pytest - -from dvc.utils.fs import makedirs - - -@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") -def test_makedirs_permissions(tmp_dir): - dir_mode = 0o755 - intermediate_dir = "тестовая-директория" - test_dir = os.path.join(intermediate_dir, "data") - - assert not os.path.exists(intermediate_dir) - - makedirs(test_dir, mode=dir_mode) - - assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode - assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 3a73375fd..1429d1550 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -8,7 +8,6 @@ from dvc.utils import relpath from dvc.utils.fs import ( BasePathNotInCheckedPathException, contains_symlink_up_to, - makedirs, path_isin, remove, ) @@ -155,5 +154,5 @@ def test_contains_symlink_case_sensitive_posix(): def test_makedirs(tmp_dir): path = os.path.join(tmp_dir, "directory") - makedirs(path) + os.makedirs(path) assert os.path.isdir(path)