ruff: ignore complex functions and remove dvc.utils.fs.makedirs (#8795)

We may want to prevent any new functions from getting complex.
This commit is contained in:
skshetry
2023-01-11 17:58:51 +05:45
committed by GitHub
parent 175c578579
commit 90a27179eb
46 changed files with 61 additions and 128 deletions
+1 -1
View File
@@ -6,7 +6,7 @@ ignore=
P1, # unindexed parameters in the str.format, see:
# https://pypi.org/project/flake8-string-format/
max_line_length = 79
max-complexity = 15
max-complexity = 10
select = B,C,E,F,W,T4,B902,T,P
show_source = true
count = true
+1 -1
View File
@@ -72,7 +72,7 @@ class CmdDataStatus(CmdBase):
yield stage, items
@classmethod
def _show_status(cls, status: "DataStatus") -> int:
def _show_status(cls, status: "DataStatus") -> int: # noqa: C901
git_info = status.pop("git") # type: ignore[misc]
result = dict(cls._process_status(status))
if not result:
+2 -2
View File
@@ -62,7 +62,7 @@ experiment_types = {
}
def _collect_rows(
def _collect_rows( # noqa: C901
base_rev,
experiments,
all_headers,
@@ -314,7 +314,7 @@ def baseline_styler(typ):
return {"style": "bold"} if typ == "baseline" else {}
def show_experiments(
def show_experiments( # noqa: C901
all_experiments,
keep=None,
drop=None,
+1 -1
View File
@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
class CmdGC(CmdBase):
def run(self):
def run(self): # noqa: C901
from dvc.repo.gc import _validate_args
_validate_args(
+1 -1
View File
@@ -94,7 +94,7 @@ class CmdPlots(CmdBase):
)
return html_template_path
def run(self):
def run(self): # noqa: C901
from pathlib import Path
from dvc.render.match import match_defs_renderers
+2 -2
View File
@@ -216,7 +216,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
{k: self._columns[k][i] for k in keys} for i in range(len(self))
]
def dropna(
def dropna( # noqa: C901
self,
axis: str = "rows",
how="any",
@@ -267,7 +267,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
else:
self.drop(*to_drop)
def drop_duplicates(
def drop_duplicates( # noqa: C901
self,
axis: str = "rows",
subset: Optional[Iterable[str]] = None,
+1 -1
View File
@@ -80,7 +80,7 @@ class AsciiCanvas:
self.canvas[y][x] = char
def line(self, x0, y0, x1, y1, char):
def line(self, x0, y0, x1, y1, char): # noqa: C901
"""Create a line on ASCII canvas.
Args:
+1 -1
View File
@@ -185,7 +185,7 @@ def _cached_clone(url, rev, for_write=False):
@wrap_with(threading.Lock())
def _clone_default_branch(url, rev, for_write=False):
def _clone_default_branch(url, rev, for_write=False): # noqa: C901
"""Get or create a clean clone of the url.
The cloned is reactualized with git pull unless rev is a known sha.
+3 -1
View File
@@ -321,7 +321,9 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
ignore_subrepos = kwargs.get("ignore_subrepos", True)
return self._info(key, path, ignore_subrepos=ignore_subrepos)
def _info(self, key, path, ignore_subrepos=True, check_ignored=True):
def _info( # noqa: C901
self, key, path, ignore_subrepos=True, check_ignored=True
):
repo, dvc_fs, subkey = self._get_subrepo_info(key)
dvc_info = None
+2 -3
View File
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Iterator, Optional
from dvc_ssh import DEFAULT_PORT, SSHFileSystem
from dvc.exceptions import DvcException
from dvc.utils.fs import makedirs
from .base import BaseMachineBackend
@@ -30,7 +29,7 @@ def _sshfs(resource: dict):
class TerraformBackend(BaseMachineBackend):
def __init__(self, tmp_dir: "StrPath", **kwargs):
super().__init__(tmp_dir, **kwargs)
makedirs(tmp_dir, exist_ok=True)
os.makedirs(tmp_dir, exist_ok=True)
@contextmanager
def make_tpi(self, name: str):
@@ -39,7 +38,7 @@ class TerraformBackend(BaseMachineBackend):
try:
working_dir = os.path.join(self.tmp_dir, name)
makedirs(working_dir, exist_ok=True)
os.makedirs(working_dir, exist_ok=True)
yield TPIBackend(working_dir=working_dir)
except TPIError as exc:
raise DvcException("TPI operation failed") from exc
+2 -2
View File
@@ -742,7 +742,7 @@ class Output:
)
return checkout_obj
def dumpd(self, **kwargs):
def dumpd(self, **kwargs): # noqa: C901
meta = self.meta.to_dict()
meta.pop("isdir", None)
ret = {**self.hash_info.to_dict(), **meta}
@@ -1034,7 +1034,7 @@ class Output:
return obj.filter(prefix)
return obj
def get_used_objs(
def get_used_objs( # noqa: C901
self, **kwargs
) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
"""Return filtered set of used object IDs for this out."""
+1 -1
View File
@@ -92,7 +92,7 @@ def _(obj: bool):
@to_str.register(dict)
def _(obj: dict):
def _(obj: dict): # noqa: C901
from dvc.config import Config
config = Config().get("parsing", {})
+1 -3
View File
@@ -10,8 +10,6 @@ from typing import List, Optional, TextIO, Union
from funcy import cached_property
from shortuuid import uuid
from dvc.utils.fs import makedirs
from .exceptions import TimeoutExpired
logger = logging.getLogger(__name__)
@@ -103,7 +101,7 @@ class ManagedProcess(AbstractContextManager):
def _make_wdir(self):
if self.wdir:
makedirs(self.wdir, exist_ok=True)
os.makedirs(self.wdir, exist_ok=True)
def _dump(self):
self._make_wdir()
+1 -1
View File
@@ -191,7 +191,7 @@ class VegaConverter(Converter):
return first(fields)
return "x"
def flat_datapoints(self, revision):
def flat_datapoints(self, revision): # noqa: C901
file2datapoints, properties = self.convert()
+1 -1
View File
@@ -113,7 +113,7 @@ def _paths_checksums(repo, targets):
return dict(_output_paths(repo, targets))
def _output_paths(repo, targets):
def _output_paths(repo, targets): # noqa: C901
from dvc.fs import LocalFileSystem
from dvc_data.hashfile.build import build
+1 -1
View File
@@ -160,7 +160,7 @@ class Experiments:
return last_applied
return None
def reproduce_celery(
def reproduce_celery( # noqa: C901
self, entries: Optional[Iterable[QueueEntry]] = None, **kwargs
) -> Dict[str, str]:
results: Dict[str, str] = {}
+1 -1
View File
@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
@locked
@scm_context
def apply(repo: "Repo", rev: str, force: bool = True, **kwargs):
def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901
from scmrepo.exceptions import SCMError as _SCMError
from dvc.repo.checkout import checkout as dvc_checkout
+3 -5
View File
@@ -102,10 +102,9 @@ class ExecutorInfo:
)
def dump_json(self, filename: str):
from dvc.utils.fs import makedirs
from dvc.utils.serialize import modify_json
makedirs(os.path.dirname(filename), exist_ok=True)
os.makedirs(os.path.dirname(filename), exist_ok=True)
with modify_json(filename) as d:
d.update(self.asdict())
@@ -288,10 +287,9 @@ class BaseExecutor(ABC):
open_func = fs.open
fs.makedirs(dpath)
else:
from dvc.utils.fs import makedirs
open_func = open
makedirs(dpath, exist_ok=True)
os.makedirs(dpath, exist_ok=True)
data = {"args": args, "kwargs": kwargs}
if extra is not None:
@@ -549,7 +547,7 @@ class BaseExecutor(ABC):
@classmethod
@contextmanager
def _repro_dvc(
def _repro_dvc( # noqa: C901
cls,
info: "ExecutorInfo",
infofile: str = None,
+2 -2
View File
@@ -22,7 +22,7 @@ from dvc.repo.experiments.refs import (
)
from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock
from dvc.scm import SCM, GitMergeError
from dvc.utils.fs import makedirs, remove
from dvc.utils.fs import remove
from .base import BaseExecutor, TaskStatus
@@ -166,7 +166,7 @@ class TempDirExecutor(BaseLocalExecutor):
**kwargs,
):
parent_dir: str = wdir or os.path.join(repo.tmp_dir, EXEC_TMP_DIR)
makedirs(parent_dir, exist_ok=True)
os.makedirs(parent_dir, exist_ok=True)
tmp_dir = mkdtemp(dir=parent_dir)
try:
executor = cls._from_stash_entry(repo, entry, tmp_dir, **kwargs)
+1 -1
View File
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
@locked
@scm_context
def pull(
def pull( # noqa: C901
repo,
git_remote: str,
exp_names: Union[Iterable[str], str],
+1 -1
View File
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
@locked
@scm_context
def push(
def push( # noqa: C901
repo,
git_remote: str,
exp_names: Union[Iterable[str], str],
+1 -1
View File
@@ -9,7 +9,7 @@ if TYPE_CHECKING:
from dvc.repo.experiments.stash import ExpStashEntry
def remove_tasks(
def remove_tasks( # noqa: C901
celery_queue: "LocalCeleryQueue",
queue_entries: Iterable["QueueEntry"],
):
+1 -1
View File
@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
@locked
@scm_context
def remove(
def remove( # noqa: C901
repo: "Repo",
exp_names: Union[None, str, List[str]] = None,
rev: Optional[str] = None,
+1 -1
View File
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
@locked
def run(
def run( # noqa: C901
repo,
targets: Optional[Iterable[str]] = None,
params: Optional[Iterable[str]] = None,
+1 -1
View File
@@ -237,7 +237,7 @@ def get_branch_names(
return names
def update_names(
def update_names( # noqa: C901
repo: "Repo",
branch_names: Dict[str, Optional[str]],
result: Dict[str, Dict[str, Any]],
+1 -1
View File
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
@locked
def fetch(
def fetch( # noqa: C901
self,
targets=None,
jobs=None,
+1 -1
View File
@@ -10,7 +10,7 @@ from . import locked
@locked
@scm_context
def imp_url(
def imp_url( # noqa: C901
self,
url,
out=None,
+2 -2
View File
@@ -1,4 +1,5 @@
import logging
import os
from tempfile import TemporaryDirectory
from typing import TYPE_CHECKING, List, Set, Tuple, Union
@@ -94,7 +95,6 @@ def save_imports(
Objects which were downloaded from source location.
"""
from dvc.stage.exceptions import DataSourceChanged
from dvc.utils.fs import makedirs
from dvc_data.index import checkout, md5, save
from dvc_objects.fs.callbacks import Callback
@@ -110,7 +110,7 @@ def save_imports(
if len(data_view):
cache = repo.odb.local
if not cache.fs.exists(cache.path):
makedirs(cache.path)
os.makedirs(cache.path)
with TemporaryDirectory(dir=cache.path) as tmpdir:
with Callback.as_tqdm_callback(
desc="Downloading imports from source",
+1 -1
View File
@@ -13,7 +13,7 @@ if TYPE_CHECKING:
@locked
def push(
def push( # noqa: C901
self,
targets=None,
jobs=None,
+2 -2
View File
@@ -86,7 +86,7 @@ def _track_stage(stage: "Stage") -> None:
@locked
@scm_context
def reproduce(
def reproduce( # noqa: C901
self: "Repo",
targets=None,
recursive=False,
@@ -138,7 +138,7 @@ def reproduce(
return _reproduce_stages(self.index.graph, list(stages), **kwargs)
def _reproduce_stages(
def _reproduce_stages( # noqa: C901
G, stages, downstream=False, single_item=False, on_unchanged=None, **kwargs
):
r"""Derive the evaluation of the given node for the given graph.
+1 -1
View File
@@ -527,7 +527,7 @@ class StageLoad:
stages_ = [stages[key] for key in keys]
return stages_, dvcfile
def _collect_all_from_repo(
def _collect_all_from_repo( # noqa: C901
self, onerror: Callable[[str, Exception], None] = None
):
"""Collects all of the stages present in the DVC repo.
+1 -1
View File
@@ -10,7 +10,7 @@ if TYPE_CHECKING:
@locked
def update(
def update( # noqa: C901
self,
targets=None,
rev=None,
+1 -1
View File
@@ -73,7 +73,7 @@ def _infos_to_str(infos):
)
def _check_blockers(tmp_dir, lock, info, *, mode, waiters):
def _check_blockers(tmp_dir, lock, info, *, mode, waiters): # noqa: C901
from .lock import LockError
non_existing_pid = set()
+1 -1
View File
@@ -195,7 +195,7 @@ def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]:
return results
def iter_revs(
def iter_revs( # noqa: C901
scm: "Git",
revs: Optional[List[str]] = None,
num: int = 1,
+2 -3
View File
@@ -52,7 +52,6 @@ from contextlib import contextmanager
from functools import partialmethod
from dvc.utils import serialize
from dvc.utils.fs import makedirs
class TmpDir(pathlib.Path):
@@ -138,11 +137,11 @@ class TmpDir(pathlib.Path):
if isinstance(contents, dict):
if not contents:
makedirs(path, exist_ok=True)
os.makedirs(path, exist_ok=True)
else:
self._gen(contents, prefix=path)
else:
makedirs(path.parent, exist_ok=True)
os.makedirs(path.parent, exist_ok=True)
if isinstance(contents, bytes):
path.write_bytes(contents)
else:
+1 -1
View File
@@ -4,7 +4,7 @@ from functools import wraps
from typing import Callable, Dict, Iterable, List, TypeVar, Union
def apply_diff(src, dest):
def apply_diff(src, dest): # noqa: C901
"""Recursively apply changes from src to dest.
Preserves dest type and hidden info in dest structure,
-35
View File
@@ -83,38 +83,3 @@ def path_isin(child: "StrPath", parent: "StrPath") -> bool:
parent = os.path.join(normalize_path(parent), "")
child = normalize_path(child)
return child != parent and child.startswith(parent)
def makedirs(path, exist_ok=False, mode=None):
if mode is None:
os.makedirs(path, exist_ok=exist_ok)
return
# Modified version of os.makedirs() with support for extended mode
# (e.g. S_ISGID)
head, tail = os.path.split(path)
if not tail:
head, tail = os.path.split(head)
if head and tail and not os.path.exists(head):
try:
makedirs(head, exist_ok=exist_ok, mode=mode)
except FileExistsError:
# Defeats race condition when another thread created the path
pass
cdir = os.curdir
if isinstance(tail, bytes):
cdir = bytes(os.curdir, "ASCII")
if tail == cdir: # xxx/newdir/. exists if xxx/newdir exists
return
try:
os.mkdir(path, mode)
except OSError:
# Cannot rely on checking for EEXIST, since the operating system
# could give priority to other errors like EACCES or EROFS
if not exist_ok or not os.path.isdir(path):
raise
try:
os.chmod(path, mode)
except OSError:
logger.trace("failed to chmod '%o' '%s'", mode, path, exc_info=True)
+1 -1
View File
@@ -87,7 +87,7 @@ class YAMLSyntaxError(PrettyDvcException, YAMLFileCorruptedError):
self.rev: Optional[str] = rev
super().__init__(self.path)
def __pretty_exc__(self, **kwargs: Any) -> None:
def __pretty_exc__(self, **kwargs: Any) -> None: # noqa: C901
from ruamel.yaml.error import MarkedYAMLError
exc = self.exc.__cause__
+1 -5
View File
@@ -162,19 +162,15 @@ line_length = 79
[tool.ruff]
line-length = 79
target-version = "py38"
format = "grouped"
show-source = true
ignore = ["S101", "SIM105", "SIM108", "SIM110", "SIM111", "SIM117", "A003", "ARG002"]
select = ["F", "E", "W", "C90", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "ISC", "ICN", "T20", "PT003", "PT008", "PT009", "PT010", "PT015", "PT016", "PT017", "PT020", "PT021", "PT024", "PT025", "PT026", "Q", "RET501", "RET504", "SIM", "ARG", "TID", "PGH", "PLC", "PLE", "PLR", "PLW", "PIE", "RUF"]
# external flake8 codes that should be preserved
external = ["B301", "B024", "C901", "E302", "C901", "F401", "W601"]
external = ["B301", "B024", "C901", "E302", "F401", "W601"]
[tool.ruff.per-file-ignores]
"scripts/**/*.py" = ["T201"]
[tool.ruff.mccabe]
max-complexity = 15
[tool.ruff.flake8-pytest-style]
fixture-parentheses = false
mark-parentheses = false
+4 -5
View File
@@ -22,7 +22,6 @@ from dvc.repo.experiments.utils import (
exp_refs_by_rev,
)
from dvc.utils import relpath
from dvc.utils.fs import makedirs
from dvc.utils.serialize import YAMLFileCorruptedError
LOCK_CONTENTS = {
@@ -447,7 +446,7 @@ def test_show_running_workspace(
"workspace",
f"workspace{BaseExecutor.INFOFILE_EXT}",
)
makedirs(os.path.dirname(pidfile), True)
os.makedirs(os.path.dirname(pidfile), exist_ok=True)
(tmp_dir / pidfile).dump_json(info.asdict())
(tmp_dir / lock_file).dump_json(LOCK_CONTENTS)
@@ -524,7 +523,7 @@ def test_show_running_tempdir(tmp_dir, scm, dvc, exp_stage, mocker):
)
info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION)
pidfile = queue.get_infofile_path(stash_rev)
makedirs(os.path.dirname(pidfile), True)
os.makedirs(os.path.dirname(pidfile), exist_ok=True)
(tmp_dir / pidfile).dump_json(info.asdict())
mock_fetch = mocker.patch.object(
dvc.experiments.tempdir_queue,
@@ -557,7 +556,7 @@ def test_show_running_celery(tmp_dir, scm, dvc, exp_stage, mocker):
)
info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION)
pidfile = queue.get_infofile_path(entries[0].stash_rev)
makedirs(os.path.dirname(pidfile), True)
os.makedirs(os.path.dirname(pidfile), exist_ok=True)
(tmp_dir / pidfile).dump_json(info.asdict())
results = dvc.experiments.show()
@@ -599,7 +598,7 @@ def test_show_running_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, mocker):
location=TempDirExecutor.DEFAULT_LOCATION,
status=TaskStatus.RUNNING,
)
makedirs(os.path.dirname(pidfile), True)
os.makedirs(os.path.dirname(pidfile), exist_ok=True)
(tmp_dir / pidfile).dump_json(info.asdict())
mocker.patch.object(
+2 -2
View File
@@ -6,7 +6,7 @@ from scmrepo.git import Git
from dvc.external_repo import CLONES, external_repo
from dvc.testing.tmp_dir import make_subrepo
from dvc.utils import relpath
from dvc.utils.fs import makedirs, remove
from dvc.utils.fs import remove
from dvc_data.hashfile.build import build
from dvc_data.hashfile.transfer import transfer
@@ -200,7 +200,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
# clear cache to test saving to cache
cache_dir = tmp_dir / repo.odb.local.path
remove(cache_dir)
makedirs(cache_dir)
os.makedirs(cache_dir)
staging, _, obj = build(
repo.odb.local,
+1 -2
View File
@@ -9,7 +9,6 @@ from dvc.odbmgr import ODBManager
from dvc.repo import Repo
from dvc.repo.get import GetDVCFileError
from dvc.testing.tmp_dir import make_subrepo
from dvc.utils.fs import makedirs
def test_get_repo_file(tmp_dir, erepo_dir):
@@ -170,7 +169,7 @@ def test_get_to_dir(tmp_dir, erepo_dir, dname):
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")
makedirs(dname, exist_ok=True)
os.makedirs(dname, exist_ok=True)
Repo.get(os.fspath(erepo_dir), "file", dname)
+2 -2
View File
@@ -13,7 +13,7 @@ from dvc.fs import system
from dvc.odbmgr import ODBManager
from dvc.stage.exceptions import StagePathNotFoundError
from dvc.testing.tmp_dir import make_subrepo
from dvc.utils.fs import makedirs, remove
from dvc.utils.fs import remove
def test_import(tmp_dir, scm, dvc, erepo_dir):
@@ -376,7 +376,7 @@ def test_download_error_pulling_imported_stage(
@pytest.mark.parametrize("dname", [".", "dir", "dir/subdir"])
def test_import_to_dir(dname, tmp_dir, dvc, erepo_dir):
makedirs(dname, exist_ok=True)
os.makedirs(dname, exist_ok=True)
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
+1 -2
View File
@@ -12,7 +12,6 @@ from dvc.dvcfile import Dvcfile
from dvc.exceptions import InvalidArgumentError
from dvc.stage import Stage
from dvc.testing.workspace_tests import TestImport as _TestImport
from dvc.utils.fs import makedirs
def test_cmd_import(tmp_dir, dvc):
@@ -74,7 +73,7 @@ def test_import_url_to_dir(dname, tmp_dir, dvc):
tmp_dir.gen({"data_dir": {"file": "file content"}})
src = os.path.join("data_dir", "file")
makedirs(dname, exist_ok=True)
os.makedirs(dname, exist_ok=True)
stage = dvc.imp_url(src, dname)
-20
View File
@@ -1,20 +0,0 @@
import os
import stat
import pytest
from dvc.utils.fs import makedirs
@pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.")
def test_makedirs_permissions(tmp_dir):
dir_mode = 0o755
intermediate_dir = "тестовая-директория"
test_dir = os.path.join(intermediate_dir, "data")
assert not os.path.exists(intermediate_dir)
makedirs(test_dir, mode=dir_mode)
assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode
assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode
+1 -2
View File
@@ -8,7 +8,6 @@ from dvc.utils import relpath
from dvc.utils.fs import (
BasePathNotInCheckedPathException,
contains_symlink_up_to,
makedirs,
path_isin,
remove,
)
@@ -155,5 +154,5 @@ def test_contains_symlink_case_sensitive_posix():
def test_makedirs(tmp_dir):
path = os.path.join(tmp_dir, "directory")
makedirs(path)
os.makedirs(path)
assert os.path.isdir(path)