Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-10 07:18:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

add ruff (#8793)

Browse Source 
* add ruff linter

* enable few more checks
This commit is contained in:
skshetry
2023-01-11 15:20:38 +05:45
committed by GitHub
parent d4bfe38a54
commit 5441964fdb
103 changed files with 359 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.pre-commit-config.yaml
+5
View File
@@ -42,6 +42,11 @@ repos:
rev: 5.10.1
hooks:
- id: isort
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: 'v0.0.218'
hooks:
- id: ruff
args: ["--force-exclude"]
- repo: https://github.com/codespell-project/codespell
rev: v2.2.2
hooks:
dvc/_debug.py
+14 -8
View File
@@ -12,7 +12,9 @@ def viztracer_profile(path: Union[Callable[[], str], str], depth: int = -1):
try:
import viztracer # pylint: disable=import-error
except ImportError:
print("Failed to run profiler, viztracer is not installed")
print( # noqa: T201
"Failed to run profiler, viztracer is not installed"
)
yield
return
@@ -34,7 +36,7 @@ def yappi_profile(
try:
import yappi # pylint: disable=import-error
except ImportError:
print("Failed to run profiler, yappi is not installed")
print("Failed to run profiler, yappi is not installed") # noqa: T201
yield
return
@@ -60,7 +62,7 @@ def yappi_profile(
st.save(out, type="callgrind")
else:
if ctx_id is not None:
print(f"\nThread {ctx_id}")
print(f"\nThread {ctx_id}") # noqa: T201
st.print_all() # pylint:disable=no-member
if ctx_id is None:
threads.print_all() # pylint:disable=no-member
@@ -74,7 +76,9 @@ def instrument(html_output=False):
try:
from pyinstrument import Profiler # pylint: disable=import-error
except ImportError:
print("Failed to run profiler, pyinstrument is not installed")
print( # noqa: T201
"Failed to run profiler, pyinstrument is not installed"
)
yield
return
@@ -87,7 +91,7 @@ def instrument(html_output=False):
if html_output:
profiler.open_in_browser()
return
print(profiler.output_text(unicode=True, color=True))
print(profiler.output_text(unicode=True, color=True)) # noqa: T201
@contextmanager
@@ -122,13 +126,15 @@ def debug():
def _sigshow(_, frame: Optional["FrameType"]) -> None:
import sys
from shutil import get_terminal_size
from traceback import format_stack
from dvc.ui import ui
lines = "\u2015" * get_terminal_size().columns
ui.error_write(lines, "\n", *format_stack(frame), lines, sep="")
file = sys.stderr
print( # noqa: T201
lines, "\n", *format_stack(frame), lines, sep="", file=file
)
@contextmanager
dvc/api/params.py
+1 -1
View File
@@ -238,7 +238,7 @@ def params_show(
if isinstance(stages, str):
stages = [stages]
def _onerror_raise(result: Dict, exception: Exception, *args, **kwargs):
def _onerror_raise(_result: Dict, exception: Exception, *args, **kwargs):
raise exception
def _postprocess(params):
dvc/cli/__init__.py
-1
View File
@@ -45,7 +45,6 @@ def _log_unknown_exceptions() -> None:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Version info for developers:\n%s", get_dvc_info())
logger.info(FOOTER)
return None
def _log_exceptions(exc: Exception) -> Optional[int]:
dvc/commands/cache.py
+1 -1
View File
@@ -42,7 +42,7 @@ def add_parser(subparsers, parent_parser):
cache_subparsers = cache_parser.add_subparsers(
dest="cmd",
help="Use `dvc cache CMD --help` for command-specific " "help.",
help="Use `dvc cache CMD --help` for command-specific help.",
)
fix_subparsers(cache_subparsers)
dvc/commands/check_ignore.py
+2 -2
View File
@@ -103,7 +103,7 @@ def add_parser(subparsers, parent_parser):
"--all",
action="store_true",
default=False,
help="Include the target paths which don’t match any pattern "
help="Include the target paths which don't match any pattern "
"in the `--details` list.",
)
parser.add_argument(
@@ -111,7 +111,7 @@ def add_parser(subparsers, parent_parser):
"--non-matching",
action="store_true",
default=False,
help="Include the target paths which don’t match any pattern "
help="Include the target paths which don't match any pattern "
"in the `--details` list.",
)
parser.add_argument(
dvc/commands/config.py
+6 -7
View File
@@ -37,13 +37,12 @@ class CmdConfig(CmdBaseNoRepo):
self.config = Config(validate=False)
def run(self):
if self.args.show_origin:
if any((self.args.value, self.args.unset)):
logger.error(
"--show-origin can't be used together with any of these "
"options: -u/--unset, value"
)
return 1
if self.args.show_origin and (self.args.value or self.args.unset):
logger.error(
"--show-origin can't be used together with any of these "
"options: -u/--unset, value"
)
return 1
if self.args.list:
return self._list()
dvc/commands/daemon.py
+1 -1
View File
@@ -46,7 +46,7 @@ def add_parser(subparsers, parent_parser):
daemon_subparsers = daemon_parser.add_subparsers(
dest="cmd",
help="Use `dvc daemon CMD --help` for command-specific " "help.",
help="Use `dvc daemon CMD --help` for command-specific help.",
)
fix_subparsers(daemon_subparsers)
dvc/commands/data.py
+3 -1
View File
@@ -114,7 +114,9 @@ class CmdDataStatus(CmdBase):
return 0
def run(self) -> int:
with log_durations(logger.trace, "in data_status"): # type: ignore
with log_durations(
logger.trace, "in data_status" # type: ignore[attr-defined]
):
status = self.repo.data_status(
granular=self.args.granular,
untracked_files=self.args.untracked_files,
dvc/commands/data_sync.py
+1 -1
View File
@@ -302,7 +302,7 @@ def add_parser(subparsers, _parent_parser):
"--with-deps",
action="store_true",
default=False,
help="Fetch cache for all dependencies of the " "specified target.",
help="Fetch cache for all dependencies of the specified target.",
)
fetch_parser.add_argument(
"-R",
dvc/commands/diff.py
+1 -1
View File
@@ -109,7 +109,7 @@ class CmdDiff(CmdBase):
ui.write()
if not sum(summary.values()):
return None
return
states_summary = ", ".join(
f"{summary[state]} {state}"
dvc/commands/experiments/show.py
+6 -4
View File
@@ -359,7 +359,7 @@ def show_experiments(
kwargs.get("iso"),
)
if keep:
for col in td.keys():
for col in td.keys(): # noqa: SIM118
if re.match(keep, col):
td.protect(col)
@@ -410,13 +410,15 @@ def show_experiments(
cols_to_drop = set()
if drop is not None:
cols_to_drop = {col for col in td.keys() if re.match(drop, col)}
cols_to_drop = {
col for col in td.keys() if re.match(drop, col) # noqa: SIM118
}
if pcp:
cols_to_drop.add("Created")
td.drop(*cols_to_drop)
if pcp:
subset = {x for x in td.keys() if x != "Experiment"}
subset = {x for x in td.keys() if x != "Experiment"} # noqa: SIM118
td.dropna(
"rows",
how="all",
@@ -494,7 +496,7 @@ class CmdExperimentsShow(CmdBase):
else DEFAULT_PRECISION
)
fill_value = "" if self.args.csv else FILL_VALUE
iso = True if self.args.csv else False
iso = self.args.csv
show_experiments(
all_experiments,
dvc/commands/machine.py
+2 -2
View File
@@ -199,7 +199,7 @@ class CmdMachineDefault(CmdMachineConfig):
if self.args.name is None and not self.args.unset:
conf = self.config.read(self.args.level)
try:
print(conf["core"]["machine"])
ui.write(conf["core"]["machine"])
except KeyError:
ui.write("No default machine set")
return 1
@@ -324,7 +324,7 @@ def add_parser(subparsers, parent_parser):
machine_subparsers = machine_parser.add_subparsers(
dest="cmd",
help="Use `dvc machine CMD --help` for " "command-specific help.",
help="Use `dvc machine CMD --help` for command-specific help.",
)
fix_subparsers(machine_subparsers)
dvc/commands/plots.py
+3 -6
View File
@@ -79,10 +79,8 @@ class CmdPlots(CmdBase):
return {k: v for k, v in props.items() if v is not None}
def _config_files(self):
config_files = None
if self.args.from_config:
config_files = {self.args.from_config}
return config_files
return {self.args.from_config}
def _html_template_path(self):
html_template_path = self.args.html_template
@@ -223,9 +221,8 @@ class CmdPlotsTemplates(CmdBase):
return 0
raise InvalidArgumentError(f"Unexpected template: {target}.")
else:
for template in TEMPLATES:
ui.write(template.DEFAULT_NAME)
for template in TEMPLATES:
ui.write(template.DEFAULT_NAME)
return 0
except DvcException:
dvc/commands/queue/__init__.py
+1 -1
View File
@@ -26,7 +26,7 @@ def add_parser(subparsers, parent_parser):
queue_subparsers = queue_parser.add_subparsers(
dest="cmd",
help="Use `dvc queue CMD --help` to display " "command-specific help.",
help="Use `dvc queue CMD --help` to display command-specific help.",
)
fix_subparsers(queue_subparsers)
dvc/commands/remote.py
+1 -1
View File
@@ -164,7 +164,7 @@ def add_parser(subparsers, parent_parser):
remote_subparsers = remote_parser.add_subparsers(
dest="cmd",
help="Use `dvc remote CMD --help` for " "command-specific help.",
help="Use `dvc remote CMD --help` for command-specific help.",
)
fix_subparsers(remote_subparsers)
dvc/commands/stage.py
+1 -1
View File
@@ -67,7 +67,7 @@ def prepare_stages_data(
class CmdStageList(CmdBase):
def _get_stages(self) -> Iterable["Stage"]:
if self.args.all:
stages: List["Stage"] = self.repo.index.stages # type: ignore
stages: List["Stage"] = self.repo.index.stages
logger.trace( # type: ignore[attr-defined]
"%d no. of stages found", len(stages)
)
dvc/compare.py
+7 -7
View File
@@ -58,7 +58,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
return self._columns[name]
def items(self) -> ItemsView[str, Column]:
projection = {k: self.column(k) for k in self.keys()}
projection = {k: self.column(k) for k in self.keys()} # noqa: SIM118
return projection.items()
def keys(self) -> List[str]:
@@ -195,7 +195,8 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
self.add_column(key)
row: List["CellT"] = [
with_value(d.get(key), self._fill_value) for key in self.keys()
with_value(d.get(key), self._fill_value)
for key in self.keys() # noqa: SIM118
]
self.append(row)
@@ -228,7 +229,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
)
if how not in ["any", "all"]:
raise ValueError(
f"Invalid 'how' value {how}." "Choose one of ['any', 'all']"
f"Invalid 'how' value {how}. Choose one of ['any', 'all']"
)
match_line: Set = set()
@@ -244,8 +245,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
if axis == "rows":
match_line.add(n_row)
break
else:
match_line.add(self.keys()[n_col])
match_line.add(self.keys()[n_col])
to_drop = match_line
if how == "all":
@@ -256,7 +256,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
to_drop -= match_line
if axis == "rows":
for name in self.keys():
for name in self.keys(): # noqa: SIM118
self._columns[name] = Column(
[
x
@@ -309,7 +309,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
else:
unique_rows.append(tuple_row)
for name in self.keys():
for name in self.keys(): # noqa: SIM118
self._columns[name] = Column(
[
x
dvc/dagascii.py
+1 -2
View File
@@ -59,8 +59,7 @@ class AsciiCanvas:
def draw(self):
"""Draws ASCII canvas on the screen."""
lines = map("".join, self.canvas)
joined_lines = os.linesep.join(lines)
return joined_lines
return os.linesep.join(lines)
def point(self, x, y, char):
"""Create a point on ASCII canvas.
dvc/dependency/base.py
+3 -5
View File
@@ -25,11 +25,9 @@ class DependencyIsStageFileError(DvcException):
class Dependency(Output):
IS_DEPENDENCY = True
DoesNotExistError = DependencyDoesNotExistError # type: Type[DvcException]
IsNotFileOrDirError = (
DependencyIsNotFileOrDirError
) # type: Type[DvcException]
IsStageFileError = DependencyIsStageFileError # type: Type[DvcException]
DoesNotExistError: Type[DvcException] = DependencyDoesNotExistError
IsNotFileOrDirError: Type[DvcException] = DependencyIsNotFileOrDirError
IsStageFileError: Type[DvcException] = DependencyIsStageFileError
def workspace_status(self) -> Dict[str, str]:
if self.fs.version_aware:
dvc/fs/__init__.py
+3 -1
View File
@@ -72,7 +72,9 @@ def get_fs_config(config, **kwargs):
except KeyError:
from dvc.config import RemoteNotFoundError
raise RemoteNotFoundError(f"remote '{name}' doesn't exist")
raise RemoteNotFoundError( # noqa: B904
f"remote '{name}' doesn't exist"
)
else:
remote_conf = kwargs
return _resolve_remote_refs(config, remote_conf)
dvc/fs/callbacks.py
+1 -1
View File
@@ -21,7 +21,7 @@ class RichCallback(Callback):
value: int = 0,
progress: "RichTransferProgress" = None,
desc: str = None,
bytes: bool = False, # pylint: disable=redefined-builtin
bytes: bool = False, # noqa: A002, pylint: disable=redefined-builtin
unit: str = None,
disable: bool = False,
transient: bool = True,
dvc/fs/dvc.py
+3 -3
View File
@@ -156,14 +156,14 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
def _get_key(self, path) -> Key:
parts = self.repo.fs.path.relparts(path, self.repo.root_dir)
if parts == (".",):
parts = ()
if parts == (os.curdir,):
return ()
return parts
def _get_key_from_relative(self, path) -> Key:
parts = self.path.relparts(path, self.root_marker)
if parts and parts[0] == os.curdir:
parts = parts[1:]
return parts[1:]
return parts
def _from_key(self, parts: Key) -> str:
dvc/ignore.py
+9 -10
View File
@@ -24,11 +24,10 @@ class DvcIgnore:
class DvcIgnorePatterns(DvcIgnore):
def __init__(self, pattern_list, dirname, sep):
if pattern_list:
if isinstance(pattern_list[0], str):
pattern_list = [
PatternInfo(pattern, "") for pattern in pattern_list
]
if pattern_list and isinstance(pattern_list[0], str):
pattern_list = [
PatternInfo(pattern, "") for pattern in pattern_list
]
self.sep = sep
self.pattern_list = pattern_list
@@ -84,7 +83,7 @@ class DvcIgnorePatterns(DvcIgnore):
return False
if os.name == "nt":
path = normalize_file(path)
return normalize_file(path)
return path
def matches(self, dirname, basename, is_dir=False, details: bool = False):
@@ -195,8 +194,8 @@ class DvcIgnoreFilter:
def _get_key(self, path):
parts = self.fs.path.relparts(path, self.root_dir)
if parts == (".",):
parts = ()
if parts == (os.curdir,):
return ()
return parts
def _update_trie(self, dirname: str, trie: Trie) -> None:
@@ -379,7 +378,7 @@ class DvcIgnoreFilter:
return False
def is_ignored_dir(self, path: str, ignore_subrepos: bool = True) -> bool:
"Only used in LocalFileSystem"
# only used in LocalFileSystem
path = self.fs.path.abspath(path)
if path == self.root_dir:
return False
@@ -387,7 +386,7 @@ class DvcIgnoreFilter:
return self._is_ignored(path, True, ignore_subrepos=ignore_subrepos)
def is_ignored_file(self, path: str, ignore_subrepos: bool = True) -> bool:
"Only used in LocalFileSystem"
# only used in LocalFileSystem
path = self.fs.path.abspath(path)
return self._is_ignored(path, False, ignore_subrepos=ignore_subrepos)
dvc/lock.py
+4 -3
View File
@@ -116,7 +116,7 @@ class Lock(LockBase):
self._lock = zc.lockfile.LockFile(self._lockfile)
except zc.lockfile.LockError:
self._lock_failed = True
raise LockError(FAILED_TO_LOCK_MESSAGE)
raise LockError(FAILED_TO_LOCK_MESSAGE) # noqa: B904
def lock(self):
retries = 6
@@ -185,14 +185,15 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
try:
super().lock(timedelta(seconds=DEFAULT_TIMEOUT))
except flufl.lock.TimeOutError:
raise LockError(FAILED_TO_LOCK_MESSAGE)
raise LockError(FAILED_TO_LOCK_MESSAGE) # noqa: B904
def _set_claimfile(self):
super()._set_claimfile()
if self._tmp_dir is not None:
# Under Windows file path length is limited so we hash it
filename = hashlib.md5(self._claimfile.encode()).hexdigest()
hasher = hashlib.md5(self._claimfile.encode()) # noqa: S324
filename = hasher.hexdigest()
self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
dvc/machine/__init__.py
+6 -2
View File
@@ -167,7 +167,9 @@ class MachineManager:
except KeyError:
from dvc.config import MachineNotFoundError
raise MachineNotFoundError(f"machine '{name}' doesn't exist")
raise MachineNotFoundError( # noqa: B904
f"machine '{name}' doesn't exist"
)
else:
conf = kwargs
return conf
@@ -179,7 +181,9 @@ class MachineManager:
backend = self.CLOUD_BACKENDS[cloud]
return self.backends[backend]
except KeyError:
raise NoMachineError(f"Machine platform '{cloud}' unsupported")
raise NoMachineError( # noqa: B904
f"Machine platform '{cloud}' unsupported"
)
def create(self, name: Optional[str]):
"""Create and start the specified machine instance."""
dvc/odbmgr.py
+1 -1
View File
@@ -39,7 +39,7 @@ class ODBManager:
from dvc.config_schema import LOCAL_COMMON
settings = {"url": config["dir"]}
for opt in LOCAL_COMMON.keys():
for opt in LOCAL_COMMON:
if opt in config:
settings[str(opt)] = config.get(opt)
dvc/output.py
+19 -20
View File
@@ -267,10 +267,10 @@ class Output:
},
)
DoesNotExistError = OutputDoesNotExistError # type: Type[DvcException]
IsNotFileOrDirError = OutputIsNotFileOrDirError # type: Type[DvcException]
IsStageFileError = OutputIsStageFileError # type: Type[DvcException]
IsIgnoredError = OutputIsIgnoredError # type: Type[DvcException]
DoesNotExistError: Type[DvcException] = OutputDoesNotExistError
IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError
IsStageFileError: Type[DvcException] = OutputIsStageFileError
IsIgnoredError: Type[DvcException] = OutputIsIgnoredError
def __init__(
self,
@@ -283,7 +283,7 @@ class Output:
persist=False,
checkpoint=False,
desc=None,
type=None, # pylint: disable=redefined-builtin
type=None, # noqa: A002, pylint: disable=redefined-builtin
labels=None,
meta=None,
remote=None,
@@ -486,10 +486,9 @@ class Output:
return self.hash_info.isdir
def _is_path_dvcignore(self, path) -> bool:
if not self.IS_DEPENDENCY and self.dvcignore:
if self.dvcignore.is_ignored(self.fs, path, ignore_subrepos=False):
return True
return False
if self.IS_DEPENDENCY or not self.dvcignore:
return False
return self.dvcignore.is_ignored(self.fs, path, ignore_subrepos=False)
@property
def exists(self):
@@ -675,11 +674,11 @@ class Output:
try:
return checkout(*args, **kwargs)
except PromptError as exc:
raise ConfirmRemoveError(exc.path)
raise ConfirmRemoveError(exc.path) # noqa: B904
except LinkError as exc:
raise CacheLinkError([exc.path])
raise CacheLinkError([exc.path]) # noqa: B904
except _CheckoutError as exc:
raise CheckoutError(exc.paths)
raise CheckoutError(exc.paths) # noqa: B904
def commit(self, filter_info=None):
if not self.exists:
@@ -762,12 +761,12 @@ class Output:
if not self.use_cache:
ret[self.PARAM_CACHE] = self.use_cache
if isinstance(self.metric, dict):
if (
self.PARAM_METRIC_XPATH in self.metric
and not self.metric[self.PARAM_METRIC_XPATH]
):
del self.metric[self.PARAM_METRIC_XPATH]
if (
isinstance(self.metric, dict)
and self.PARAM_METRIC_XPATH in self.metric
and not self.metric[self.PARAM_METRIC_XPATH]
):
del self.metric[self.PARAM_METRIC_XPATH]
if self.metric:
ret[self.PARAM_METRIC] = self.metric
@@ -1020,7 +1019,7 @@ class Output:
"Would you like to continue? Use '-f' to force."
)
if not force and not prompt.confirm(msg.format(self.fs_path)):
raise CollectCacheError(
raise CollectCacheError( # noqa: B904
"unable to fully collect used cache"
" without cache for directory '{}'".format(self)
)
@@ -1032,7 +1031,7 @@ class Output:
prefix = self.fs.path.parts(
self.fs.path.relpath(filter_info, self.fs_path)
)
obj = obj.filter(prefix)
return obj.filter(prefix)
return obj
def get_used_objs(
dvc/parsing/context.py
+8 -8
View File
@@ -414,19 +414,19 @@ class Context(CtxDict):
self.imports[path].extend(cp)
def check_loaded(self, path, item, keys):
if not keys and isinstance(self.imports[path], list):
imported = self.imports[path]
if not keys and isinstance(imported, list):
raise VarsAlreadyLoaded(
f"cannot load '{item}' as it's partially loaded already"
)
elif keys and self.imports[path] is None:
if keys and imported is None:
raise VarsAlreadyLoaded(
f"cannot partially load '{item}' as it's already loaded."
)
elif isinstance(self.imports[path], list):
if not set(keys).isdisjoint(set(self.imports[path])):
raise VarsAlreadyLoaded(
f"cannot load '{item}' as it's partially loaded already"
)
if isinstance(imported, list) and set(keys) & set(imported):
raise VarsAlreadyLoaded(
f"cannot load '{item}' as it's partially loaded already"
)
def load_from_vars(
self,
@@ -482,7 +482,7 @@ class Context(CtxDict):
try:
yield
finally:
for key in new.keys():
for key in new:
self._reserved_keys.pop(key)
@contextmanager
dvc/parsing/interpolate.py
+3 -4
View File
@@ -186,7 +186,7 @@ def parse_expr(s: str):
result = get_parser().parseString(s, parseAll=True)
except ParseException as exc:
format_and_raise_parse_error(exc)
raise AssertionError("unreachable")
raise AssertionError("unreachable") # noqa: B904
joined = result.asList()
assert len(joined) == 1
@@ -204,9 +204,8 @@ def validate_value(value, key):
not_primitive = value is not None and not isinstance(value, PRIMITIVES)
not_foreach = key is not None and "foreach" not in key
if not_primitive and not_foreach:
if isinstance(value, dict):
if key == "cmd":
return True
if isinstance(value, dict) and key == "cmd":
return True
raise ParseError(
f"Cannot interpolate data of type '{type(value).__name__}'"
)
dvc/pathspec_math.py
+1 -2
View File
@@ -50,8 +50,7 @@ def change_rule(rule, rel):
rule = f"!/{rel}{rule}"
else:
rule = f"/{rel}{rule}"
rule = normalize_file(rule)
return rule
return normalize_file(rule)
def _change_dirname(dirname, pattern_list, new_dirname):
dvc/proc/manager.py
+10 -12
View File
@@ -38,7 +38,7 @@ class ProcessManager:
with open(info_path, encoding="utf-8") as fobj:
return ProcessInfo.from_dict(json.load(fobj))
except FileNotFoundError:
raise KeyError
raise KeyError # noqa: B904
@reraise(FileNotFoundError, KeyError)
def __setitem__(self, key: str, value: "ProcessInfo"):
@@ -83,13 +83,12 @@ class ProcessManager:
def send_signal(self, name: str, sig: int):
"""Send `signal` to the specified named process."""
process_info = self[name]
if sys.platform == "win32":
if sig not in (
signal.SIGTERM,
signal.CTRL_C_EVENT,
signal.CTRL_BREAK_EVENT,
):
raise UnsupportedSignalError(sig)
if sys.platform == "win32" and sig not in (
signal.SIGTERM,
signal.CTRL_C_EVENT, # pylint: disable=no-member
signal.CTRL_BREAK_EVENT, # pylint: disable=no-member
):
raise UnsupportedSignalError(sig)
def handle_closed_process():
logging.warning(
@@ -105,10 +104,9 @@ class ProcessManager:
handle_closed_process()
raise
except OSError as exc:
if sys.platform == "win32":
if exc.winerror == 87:
handle_closed_process()
raise ProcessLookupError from exc
if getattr(exc, "winerror", None) == 87:
handle_closed_process()
raise ProcessLookupError from exc
raise
def terminate(self, name: str):
dvc/progress.py
+1 -1
View File
@@ -46,7 +46,7 @@ class Tqdm(tqdm):
desc=None,
leave=False,
bar_format=None,
bytes=False, # pylint: disable=redefined-builtin
bytes=False, # noqa: A002, pylint: disable=redefined-builtin
file=None,
total=None,
postfix=None,
dvc/render/converter/vega.py
+25 -25
View File
@@ -161,35 +161,35 @@ class VegaConverter(Converter):
@staticmethod
def infer_y_label(properties):
y_label = properties.get("y_label", None)
if y_label is None:
y = properties.get("y", None)
if isinstance(y, dict):
fields = {field for _, field in _file_field(y)}
if len(fields) == 1:
y_label = first(fields)
else:
y_label = "y"
elif isinstance(y, list):
y_label = "y"
elif isinstance(y, str):
y_label = y
if y_label is not None:
return y_label
y = properties.get("y", None)
if isinstance(y, str):
return y
if isinstance(y, list):
return "y"
if not isinstance(y, dict):
return
return y_label
fields = {field for _, field in _file_field(y)}
if len(fields) == 1:
return first(fields)
return "y"
@staticmethod
def infer_x_label(properties):
x_label = properties.get("x_label", None)
if x_label is None:
x = properties.get("x", None)
if isinstance(x, dict):
fields = {field for _, field in _file_field(x)}
if len(fields) == 1:
x_label = first(fields)
else:
x_label = "x"
else:
x_label = INDEX_FIELD
return x_label
if x_label is not None:
return x_label
x = properties.get("x", None)
if not isinstance(x, dict):
return INDEX_FIELD
fields = {field for _, field in _file_field(x)}
if len(fields) == 1:
return first(fields)
return "x"
def flat_datapoints(self, revision):
@@ -251,7 +251,7 @@ class VegaConverter(Converter):
source_datapoints=x_datapoints,
)
except IndexError:
raise DvcException(
raise DvcException( # noqa: B904
f"Cannot join '{x_field}' from '{x_file}' and "
f"'{y_field}' from '{y_file}'. "
"They have to have same length."
dvc/repo/add.py
+4 -2
View File
@@ -91,9 +91,11 @@ def translate_graph_error(stages: Stages) -> Iterator[None]:
parent=exc.parent.fs_path,
parent_stage=exc.parent.stage.addressing,
)
raise OverlappingOutputPathsError(exc.parent, exc.overlapping_out, msg)
raise OverlappingOutputPathsError( # noqa: B904
exc.parent, exc.overlapping_out, msg
)
except OutputDuplicationError as exc:
raise OutputDuplicationError(
raise OutputDuplicationError( # noqa: B904
exc.output, list(set(exc.stages) - set(stages))
)
dvc/repo/experiments/apply.py
+2 -2
View File
@@ -47,7 +47,7 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs):
baseline_sha = repo.scm.get_rev()
if exp_ref_info:
if baseline_sha != exp_ref_info.baseline_sha:
raise InvalidExpRevError(rev)
raise InvalidExpRevError(rev) # noqa: B904
exp_rev = repo.scm.get_ref(str(exp_ref_info))
elif queue_entry:
if queue_entry.baseline_rev != baseline_sha:
@@ -65,7 +65,7 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs):
try:
repo.scm.merge(exp_rev, commit=False, squash=True)
except _SCMError as exc:
raise GitMergeError(str(exc), scm=repo.scm)
raise GitMergeError(str(exc), scm=repo.scm) # noqa: B904
repo.scm.reset()
dvc/repo/experiments/branch.py
+1 -1
View File
@@ -19,7 +19,7 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs):
try:
rev = resolve_rev(repo.scm, exp_rev)
except RevError:
raise InvalidArgumentError(exp_rev)
raise InvalidArgumentError(exp_rev) # noqa: B904
ref_info = None
ref_infos = list(exp_refs_by_rev(repo.scm, rev))
dvc/repo/experiments/executor/base.py
+5 -7
View File
@@ -77,9 +77,8 @@ class ExecutorInfo:
@classmethod
def from_dict(cls, d):
if "collected" in d:
if d.pop("collected"):
d["status"] = TaskStatus.FINISHED
if d.pop("collected", None):
d["status"] = TaskStatus.FINISHED
return cls(**d)
def asdict(self):
@@ -247,7 +246,7 @@ class BaseExecutor(ABC):
root_dir: str,
**kwargs,
) -> _T:
executor = cls(
return cls(
root_dir=root_dir,
dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
baseline_rev=entry.baseline_rev,
@@ -256,7 +255,6 @@ class BaseExecutor(ABC):
wdir=relpath(os.getcwd(), repo.scm.root_dir),
**kwargs,
)
return executor
@staticmethod
def hash_exp(stages: Iterable["PipelineStage"]) -> str:
@@ -379,7 +377,7 @@ class BaseExecutor(ABC):
try:
dvc.scm.validate_git_remote(git_remote)
except InvalidRemote as exc:
raise InvalidRemoteSCMRepo(str(exc))
raise InvalidRemoteSCMRepo(str(exc)) # noqa: B904
dvc.cloud.get_remote_odb()
@classmethod
@@ -618,7 +616,7 @@ class BaseExecutor(ABC):
push_cache=push_cache,
run_cache=run_cache,
)
except BaseException as exc: # pylint: disable=broad-except
except BaseException as exc: # noqa: BLE001, pylint: disable=W0703
logger.warning(
"Something went wrong while auto pushing experiment "
"to the remote '%s': %s",
dvc/repo/experiments/executor/local.py
+2 -2
View File
@@ -137,7 +137,7 @@ class TempDirExecutor(BaseLocalExecutor):
try:
self.scm.merge(merge_rev, squash=True, commit=False)
except _SCMError as exc:
raise GitMergeError(str(exc), scm=self.scm)
raise GitMergeError(str(exc), scm=self.scm) # noqa: B904
def _config(self, cache_dir):
local_config = os.path.join(
@@ -227,7 +227,7 @@ class WorkspaceExecutor(BaseLocalExecutor):
try:
self.scm.merge(merge_rev, squash=True, commit=False)
except _SCMError as exc:
raise GitMergeError(str(exc), scm=self.scm)
raise GitMergeError(str(exc), scm=self.scm) # noqa: B904
if branch:
self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
elif scm.get_ref(EXEC_BRANCH):
dvc/repo/experiments/executor/ssh.py
+3 -4
View File
@@ -112,11 +112,10 @@ class SSHExecutor(BaseExecutor):
@staticmethod
def _git_client_args(fs):
kwargs = {
return {
"password": fs.fs_args.get("password"),
"key_filename": first(fs.fs_args.get("client_keys", [])),
}
return kwargs
def init_git(
self,
@@ -243,8 +242,8 @@ class SSHExecutor(BaseExecutor):
def reproduce(
cls,
info: "ExecutorInfo",
rev: str,
queue: Optional["Queue"] = None,
rev: str, # noqa: ARG003
queue: Optional["Queue"] = None, # noqa: ARG003
infofile: Optional[str] = None,
log_errors: bool = True,
log_level: Optional[int] = None,
dvc/repo/experiments/init.py
+8 -7
View File
@@ -142,14 +142,13 @@ def validate_prompts(
except MissingParamsFile:
return value, msg_format.format(value, "file")
except ParamsIsADirectoryError:
raise InvalidResponse(
raise InvalidResponse( # noqa: B904
f"[prompt.invalid]'{value}' is a directory. "
"Please retry with an existing parameters file."
)
elif key in ("code", "data"):
if not os.path.exists(value):
typ = "file" if is_file(value) else "directory"
return value, msg_format.format(value, typ)
elif key in ("code", "data") and not os.path.exists(value):
typ = "file" if is_file(value) else "directory"
return value, msg_format.format(value, typ)
return value
@@ -200,7 +199,7 @@ def init_out_dirs(stage: PipelineStage) -> List[str]:
def init(
repo: "Repo",
name: str = "train",
type: str = "default", # pylint: disable=redefined-builtin
type: str = "default", # noqa: A002, pylint: disable=redefined-builtin
defaults: Dict[str, str] = None,
overrides: Dict[str, str] = None,
interactive: bool = False,
@@ -244,7 +243,9 @@ def init(
try:
ParamsDependency(None, params, repo=repo).validate_filepath()
except ParamsIsADirectoryError as exc:
raise DvcException(f"{exc}.") # swallow cause for display
raise DvcException( # noqa: B904
f"{exc}."
) # swallow cause for display
except MissingParamsFile:
pass
dvc/repo/experiments/push.py
+1 -1
View File
@@ -96,7 +96,7 @@ def _push(
progress=pbar.update_git,
)
except AuthError as exc:
raise GitAuthError(str(exc))
raise GitAuthError(str(exc)) # noqa: B904
def group_result(refspec):
return results[str(refspec)]
dvc/repo/experiments/queue/base.py
+1 -1
View File
@@ -508,7 +508,7 @@ class BaseStashQueue(ABC):
with open(self.args_file, "rb") as fobj:
try:
data = pickle.load(fobj)
except Exception: # pylint: disable=broad-except
except Exception: # noqa: BLE001, pylint: disable=broad-except
data = {}
extra = int(data.get("extra", 0)) + 1
else:
dvc/repo/experiments/queue/celery.py
+6 -4
View File
@@ -72,7 +72,7 @@ class LocalCeleryQueue(BaseStashQueue):
@cached_property
def celery(self) -> "FSApp":
from kombu.transport.filesystem import Channel # type: ignore
from kombu.transport.filesystem import Channel
# related to https://github.com/iterative/dvc-task/issues/61
Channel.QoS.restore_at_shutdown = False
@@ -232,10 +232,10 @@ class LocalCeleryQueue(BaseStashQueue):
exp_result = self.get_result(entry)
except FileNotFoundError:
if result.status == "SUCCESS":
raise DvcException(
raise DvcException( # noqa: B904
f"Invalid experiment '{entry.stash_rev[:7]}'."
)
elif result.status == "FAILURE":
if result.status == "FAILURE":
exp_result = None
yield QueueDoneResult(entry, exp_result)
@@ -421,7 +421,9 @@ class LocalCeleryQueue(BaseStashQueue):
try:
proc_info = self.proc[queue_entry.stash_rev]
except KeyError:
raise DvcException(f"No output logs found for experiment '{rev}'")
raise DvcException( # noqa: B904
f"No output logs found for experiment '{rev}'"
)
with open(
proc_info.stdout,
encoding=encoding or locale.getpreferredencoding(),
dvc/repo/experiments/queue/tasks.py
+2 -2
View File
@@ -43,7 +43,7 @@ def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
@shared_task
def collect_exp(
proc_dict: Dict[str, Any], # pylint: disable=unused-argument
proc_dict: Dict[str, Any], # noqa: ARG001, pylint: disable=unused-argument
entry_dict: Dict[str, Any],
) -> str:
"""Collect results for an experiment.
@@ -73,7 +73,7 @@ def collect_exp(
else:
logger.debug("Experiment failed (Exec result was None)")
celery_queue.stash_failed(entry)
except Exception: # pylint: disable=broad-except
except Exception: # noqa: BLE001, pylint: disable=broad-except
# Log exceptions but do not re-raise so that task chain execution
# continues
logger.exception("Failed to collect experiment")
dvc/repo/experiments/queue/tempdir.py
+1 -1
View File
@@ -130,7 +130,7 @@ class TempDirQueue(WorkspaceQueue):
return results
except DvcException:
raise
except Exception as exc:
except Exception as exc: # noqa: BLE001
raise DvcException(
f"Failed to reproduce experiment '{rev[:7]}'"
) from exc
dvc/repo/experiments/queue/workspace.py
+2 -2
View File
@@ -134,7 +134,7 @@ class WorkspaceQueue(BaseStashQueue):
return {}
except DvcException:
raise
except Exception as exc:
except Exception as exc: # noqa: BLE001
raise DvcException(
f"Failed to reproduce experiment '{rev[:7]}'"
) from exc
@@ -152,7 +152,7 @@ class WorkspaceQueue(BaseStashQueue):
@staticmethod
def collect_executor( # pylint: disable=unused-argument
exp: "Experiments",
executor: BaseExecutor,
executor: BaseExecutor, # noqa: ARG004
exec_result: ExecutorResult,
) -> Dict[str, str]:
results: Dict[str, str] = {}
dvc/repo/experiments/refs.py
+1 -1
View File
@@ -55,7 +55,7 @@ class ExpRefInfo:
):
raise InvalidExpRefError(ref)
except ValueError:
raise InvalidExpRefError(ref)
raise InvalidExpRefError(ref) # noqa: B904
baseline_sha = parts[2] + parts[3]
name = parts[4] if len(parts) == 5 else None
return cls(baseline_sha, name)
dvc/repo/experiments/show.py
+2 -3
View File
@@ -43,8 +43,7 @@ class ExpStatus(Enum):
def _is_scm_error(collected_exp: Dict[str, Any]) -> bool:
if "error" in collected_exp and (
isinstance(collected_exp["error"], SCMError)
or isinstance(collected_exp["error"], InnerScmError)
isinstance(collected_exp["error"], (SCMError, InnerScmError))
):
return True
return False
@@ -135,7 +134,7 @@ def _collect_complete_experiment(
) -> Dict[str, Dict[str, Any]]:
results: "OrderedDict[str, Dict[str, Any]]" = OrderedDict()
checkpoint: bool = True if len(revs) > 1 else False
checkpoint = len(revs) > 1
prev = ""
for rev in revs:
dvc/repo/experiments/utils.py
+5 -5
View File
File diff suppressed because one or more lines are too long
dvc/repo/imp_url.py
+6 -8
View File
@@ -1,5 +1,4 @@
import os
from typing import TYPE_CHECKING
from dvc.repo.scm_context import scm_context
from dvc.utils import relpath, resolve_output, resolve_paths
@@ -8,9 +7,6 @@ from dvc.utils.fs import path_isin
from ..exceptions import InvalidArgumentError, OutputDuplicationError
from . import locked
if TYPE_CHECKING:
from dvc.dvcfile import DVCFile
@locked
@scm_context
@@ -26,14 +22,14 @@ def imp_url(
remote=None,
to_remote=False,
desc=None,
type=None, # pylint: disable=redefined-builtin
type=None, # noqa: A002, pylint: disable=redefined-builtin
labels=None,
meta=None,
jobs=None,
fs_config=None,
version_aware: bool = False,
):
from dvc.dvcfile import Dvcfile
from dvc.dvcfile import make_dvcfile
from dvc.stage import Stage, create_stage, restore_fields
out = resolve_output(url, out)
@@ -78,14 +74,16 @@ def imp_url(
out_obj = stage.outs[0]
out_obj.annot.update(desc=desc, type=type, labels=labels, meta=meta)
dvcfile: "DVCFile" = Dvcfile(self, stage.path) # type: ignore
dvcfile = make_dvcfile(self, stage.path)
dvcfile.remove()
try:
new_index = self.index.add(stage)
new_index.check_graph()
except OutputDuplicationError as exc:
raise OutputDuplicationError(exc.output, set(exc.stages) - {stage})
raise OutputDuplicationError( # noqa: B904
exc.output, set(exc.stages) - {stage}
)
if no_exec:
stage.ignore_outs()
dvc/repo/init.py
+1 -1
View File
@@ -43,7 +43,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
try:
scm = SCM(root_dir, search_parent_directories=subdir, no_scm=no_scm)
except SCMError:
raise InitError(
raise InitError( # noqa: B904
f"{root_dir} is not tracked by any supported SCM tool (e.g. Git). "
"Use `--no-scm` if you don't want to use any SCM or "
"`--subdir` if initializing inside a subdirectory of a parent SCM "
dvc/repo/install.py
+3 -1
View File
@@ -54,7 +54,9 @@ def install_hooks(scm: "Git") -> None:
scm.verify_hook(hook)
except GitHookAlreadyExists as exc:
link = format_link("https://man.dvc.org/install")
raise DvcException(f"{exc}. Please refer to {link} for more info.")
raise DvcException( # noqa: B904
f"{exc}. Please refer to {link} for more info."
)
for hook in hooks:
scm.install_hook(hook, f"exec dvc git-hook {hook} $@")
dvc/repo/ls.py
+1 -1
View File
@@ -61,7 +61,7 @@ def _ls(
try:
fs_path = fs.info(fs_path)["name"]
except FileNotFoundError:
raise PathMissingError(path, repo, dvc_only=dvc_only)
raise PathMissingError(path, repo, dvc_only=dvc_only) # noqa: B904
infos = {}
for root, dirs, files in fs.walk(
dvc/repo/remove.py
+1 -2
View File
@@ -26,8 +26,7 @@ def remove(self: "Repo", target: str, outs: bool = False):
# give a more helpful error message.
if self.fs.exists(target + DVC_FILE_SUFFIX):
raise StageFileIsNotDvcFileError(target) from e
else:
raise
raise
for stage in stages:
stage.remove(remove_outs=outs, force=outs)
dvc/repo/reproduce.py
+1 -1
View File
@@ -222,7 +222,7 @@ def _reproduce_stages(
)
break
except Exception as exc:
except Exception as exc: # noqa: BLE001
raise ReproductionError(stage.addressing) from exc
if on_unchanged is not None:
dvc/repo/scm_context.py
+3 -5
View File
@@ -78,7 +78,7 @@ class SCMContext:
try:
gitignore_file = self.scm.ignore(path)
except FileNotInRepoError as exc:
raise SCMError(str(exc))
raise SCMError(str(exc)) # noqa: B904
if gitignore_file:
logger.debug("Added '%s' to gitignore file.", path)
@@ -94,7 +94,7 @@ class SCMContext:
try:
gitignore_file = self.scm.ignore_remove(path)
except FileNotInRepoError as exc:
raise SCMError(str(exc))
raise SCMError(str(exc)) # noqa: B904
if gitignore_file:
return self.track_file(relpath(gitignore_file))
@@ -130,9 +130,7 @@ class SCMContext:
and logger.isEnabledFor(logging.INFO)
):
add_cmd = self._make_git_add_cmd(self.files_to_track)
logger.info(
f"\nTo track the changes with git, run:\n" f"\n{add_cmd}"
)
logger.info(f"\nTo track the changes with git, run:\n\n{add_cmd}")
logger.info(
"\nTo enable auto staging, run:\n\n"
"\tdvc config core.autostage true"
dvc/repo/stage.py
+3 -3
View File
@@ -289,7 +289,7 @@ class StageLoad:
path = self._get_filepath(path, name)
dvcfile = Dvcfile(self.repo, path)
# `dvcfile.stages` is not cached
stages = dvcfile.stages # type: ignore
stages = dvcfile.stages # type: ignore[attr-defined]
if isinstance(stages, SingleStageLoader):
stage = stages[name]
@@ -311,7 +311,7 @@ class StageLoad:
path = self._get_filepath(path, name)
dvcfile = Dvcfile(self.repo, path)
stages = dvcfile.stages # type: ignore
stages = dvcfile.stages # type: ignore[attr-defined]
return stages[name]
@@ -517,7 +517,7 @@ class StageLoad:
path = self._get_filepath(path)
dvcfile = Dvcfile(self.repo, path)
stages = dvcfile.stages # type: ignore
stages = dvcfile.stages
if isinstance(stages, SingleStageLoader):
stages_ = [stages[None]]
dvc/repo/update.py
+8 -5
View File
@@ -34,11 +34,14 @@ def update(
"--to-remote can't be used with --no-download"
)
if not to_remote and remote:
if not self.cloud.get_remote(name=remote).worktree:
raise InvalidArgumentError(
"--remote can't be used without --to-remote"
)
if (
not to_remote
and remote
and not self.cloud.get_remote(name=remote).worktree
):
raise InvalidArgumentError(
"--remote can't be used without --to-remote"
)
import_stages = set()
other_stage_infos: List["StageInfo"] = []
dvc/repo/worktree.py
+12 -9
View File
@@ -232,17 +232,20 @@ def update_worktree_stages(
)
continue
entry = remote_index[key]
if entry.meta and entry.meta.isdir:
if not any(
if (
entry.meta
and entry.meta.isdir
and not any(
subkey != key and subentry.meta and not subentry.meta.isdir
for subkey, subentry in remote_index.iteritems(key)
):
logger.warning(
"Could not update '%s', directory is empty in the "
"remote",
out,
)
continue
)
):
logger.warning(
"Could not update '%s', directory is empty in the "
"remote",
out,
)
continue
_fetch_out_changes(out, local_index, remote_index, remote)
stage.save(merge_versioned=True)
for out in stage.outs:
dvc/schema.py
+1 -1
View File
@@ -46,7 +46,7 @@ LOCKFILE_V2_SCHEMA = {
OUT_PSTAGE_DETAILED_SCHEMA = {
str: {
**ANNOTATION_SCHEMA, # type: ignore
**ANNOTATION_SCHEMA, # type: ignore[arg-type]
Output.PARAM_CACHE: bool,
Output.PARAM_PERSIST: bool,
Output.PARAM_CHECKPOINT: bool,
dvc/scm.py
+2 -2
View File
@@ -169,11 +169,11 @@ def resolve_rev(scm: "Git", rev: str) -> str:
try:
ref_infos = resolve_name(scm, rev).get(rev)
except AmbiguousExpRefInfo:
raise RevError(f"ambiguous Git revision '{rev}'")
raise RevError(f"ambiguous Git revision '{rev}'") # noqa: B904
if ref_infos:
return scm.get_ref(str(ref_infos))
raise RevError(str(exc))
raise RevError(str(exc)) # noqa: B904
def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]:
dvc/stage/__init__.py
+1 -1
View File
@@ -276,7 +276,7 @@ class Stage(params.StageParams):
with suppress(ValueError):
# try to use first non-empty line as a description
line = next(filter(None, self.desc.splitlines()))
desc = line.strip()
return line.strip()
return desc
def _read_env(self, out, checkpoint_func=None) -> Env:
dvc/stage/loader.py
+1 -1
View File
@@ -126,7 +126,7 @@ class StageLoader(Mapping):
try:
resolved_data = self.resolver.resolve_one(name)
except EntryNotFound:
raise StageNotFound(self.dvcfile, name)
raise StageNotFound(self.dvcfile, name) # noqa: B904
if self.lockfile_data and name not in self.lockfile_data:
self.lockfile_needs_update()
dvc/stage/monitor.py
+1 -1
View File
@@ -110,7 +110,7 @@ class Monitor:
try:
task.execute()
task.updated.set()
except Exception: # pylint: disable=broad-except
except Exception: # noqa: BLE001, pylint: disable=W0703
logger.exception(
"Error running '%s' task, '%s' will be aborted",
task.name,
dvc/stage/run.py
+1 -1
View File
@@ -66,7 +66,7 @@ def prepare_kwargs(stage, checkpoint_func=None, run_env=None):
# #L1426
# [2] https://github.com/iterative/dvc/issues/2506
# #issuecomment-535396799
kwargs["shell"] = True if os.name == "nt" else False
kwargs["shell"] = os.name == "nt"
return kwargs
dvc/stage/serialize.py
+6 -5
View File
@@ -201,8 +201,9 @@ def to_single_stage_file(stage: "Stage", **kwargs):
# - apply changes to a returned structure
# - serialize it
text = stage._stage_text # noqa, pylint: disable=protected-access
if text is not None:
saved_state = parse_yaml_for_update(text, stage.path)
apply_diff(state, saved_state)
state = saved_state
return state
if text is None:
return state
saved_state = parse_yaml_for_update(text, stage.path)
apply_diff(state, saved_state)
return saved_state
dvc/stage/utils.py
+1 -1
View File
@@ -273,7 +273,7 @@ def check_stage_exists(
raise StageFileAlreadyExistsError(
f"'{stage.relpath}' already exists. {hint}"
)
elif stage.name and stage.name in dvcfile.stages:
if stage.name and stage.name in dvcfile.stages:
raise DuplicateStageName(
f"Stage '{stage.name}' already exists in '{stage.relpath}'. {hint}"
)
dvc/testing/fixtures.py
+7 -6
View File
@@ -1,5 +1,6 @@
import os
import subprocess
from typing import Dict, Tuple
import pytest
@@ -24,7 +25,7 @@ __all__ = [
"docker_services",
]
CACHE = {} # type: ignore
CACHE: Dict[Tuple[bool, bool, bool], str] = {}
@pytest.fixture(scope="session")
@@ -42,9 +43,9 @@ def make_tmp_dir(tmp_path_factory, request, worker_id):
cache = CACHE.get((scm, dvc, subdir))
if not cache:
cache = tmp_path_factory.mktemp("dvc-test-cache" + worker_id)
TmpDir(cache).init(scm=scm, dvc=dvc, subdir=subdir)
CACHE[(scm, dvc, subdir)] = os.fspath(cache)
cache_dir = tmp_path_factory.mktemp("dvc-test-cache" + worker_id)
TmpDir(cache_dir).init(scm=scm, dvc=dvc, subdir=subdir)
CACHE[(scm, dvc, subdir)] = cache = os.fspath(cache_dir)
path = tmp_path_factory.mktemp(name) if isinstance(name, str) else name
# ignore sqlite files from .dvc/tmp. We might not be closing the cache
@@ -111,7 +112,7 @@ def local_cloud(make_cloud):
@pytest.fixture
def make_remote(tmp_dir, dvc, make_cloud):
def make_remote(tmp_dir, dvc, make_cloud): # noqa: ARG001
def _make_remote(name, typ="local", **kwargs):
cloud = make_cloud(typ) # pylint: disable=W0621
tmp_dir.add_remote(name=name, config=cloud.config, **kwargs)
@@ -179,7 +180,7 @@ def docker():
@pytest.fixture(scope="session")
def docker_compose(docker):
def docker_compose(docker): # noqa: ARG001
try:
subprocess.check_output("docker-compose version", shell=True)
except (subprocess.CalledProcessError, OSError):
dvc/ui/__init__.py
+1 -1
View File
@@ -279,7 +279,7 @@ class Console:
if not opened:
ui.error_write(
f"Failed to open {url}. " "Please try opening it manually."
f"Failed to open {url}. Please try opening it manually."
)
return 1
dvc/ui/pager.py
+1 -1
View File
@@ -73,7 +73,7 @@ def find_pager():
# `-R` is required to render ansi escape sequences for exp show
# and, `-S` is required for horizontal scrolling.
less_env = bool(os.getenv(LESS))
pager = prepare_default_pager(
return prepare_default_pager(
ansi_escapes=True,
chop_long_lines=True,
quit_if_one_screen=not less_env,
dvc/updater.py
+2 -2
View File
@@ -77,9 +77,9 @@ class Updater:
try:
info = json.load(fobj)
latest = info["version"]
except Exception as exc: # pylint: disable=broad-except
except Exception as e: # noqa: BLE001 # pylint: disable=W0703
logger.debug(
"'%s' is not a valid json: %s", self.updater_file, exc
"'%s' is not a valid json: %s", self.updater_file, e
)
self.fetch()
return
dvc/utils/__init__.py
+2 -4
View File
@@ -196,14 +196,12 @@ def boxify(message, border_color=None):
for line in lines
]
box_str = "{margin}{padding}{content}{padding}{margin}".format(
return "{margin}{padding}{content}{padding}{margin}".format(
margin=colorize(margin, color=border_color),
padding="".join(padding_lines),
content="".join(content_lines),
)
return box_str
def _visual_width(line):
"""Get the the number of columns required to display a string"""
@@ -410,7 +408,7 @@ def error_handler(func):
vals = func(*args, **kwargs)
if vals:
result["data"] = vals
except Exception as e: # pylint: disable=broad-except
except Exception as e: # noqa: BLE001, pylint: disable=broad-except
if onerror is not None:
onerror(result, e, **kwargs)
return result
dvc/utils/collections.py
+5 -6
View File
@@ -58,9 +58,9 @@ def to_omegaconf(item):
Cast the custom classes to Python primitives.
"""
if isinstance(item, dict):
item = {k: to_omegaconf(v) for k, v in item.items()}
elif isinstance(item, list):
item = [to_omegaconf(x) for x in item]
return {k: to_omegaconf(v) for k, v in item.items()}
if isinstance(item, list):
return [to_omegaconf(x) for x in item]
return item
@@ -177,7 +177,6 @@ def nested_contains(dictionary: Dict, phrase: str) -> bool:
if key == phrase and val:
return True
if isinstance(val, dict):
if nested_contains(val, phrase):
return True
if isinstance(val, dict) and nested_contains(val, phrase):
return True
return False
dvc/utils/fs.py
+1 -1
View File
@@ -40,7 +40,7 @@ def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"):
return contains_symlink_up_to(os.path.dirname(path), base_path)
def _chmod(func, p, excinfo): # pylint: disable=unused-argument
def _chmod(func, p, excinfo): # noqa: ARG001, pylint: disable=unused-argument
perm = os.lstat(p).st_mode
perm |= stat.S_IWRITE
dvc/utils/serialize/_common.py
+3 -3
View File
@@ -65,7 +65,7 @@ class EncodingError(ParseError):
def _load_data(path: "AnyPath", parser: ParserFn, fs: "FileSystem" = None):
open_fn = fs.open if fs else open
encoding = "utf-8"
with open_fn(path, encoding=encoding) as fd: # type: ignore
with open_fn(path, encoding=encoding) as fd:
with reraise(UnicodeDecodeError, EncodingError(path, encoding)):
return parser(fd.read(), path)
@@ -78,7 +78,7 @@ def _dump_data(
**dumper_args,
):
open_fn = fs.open if fs else open
with open_fn(path, "w+", encoding="utf-8") as fd: # type: ignore
with open_fn(path, "w+", encoding="utf-8") as fd:
dumper(data, fd, **dumper_args)
@@ -90,7 +90,7 @@ def _modify_data(
fs: "FileSystem" = None,
):
exists_fn = fs.exists if fs else os.path.exists
file_exists = exists_fn(path) # type: ignore
file_exists = exists_fn(path)
data = _load_data(path, parser=parser, fs=fs) if file_exists else {}
yield data
dumper(path, data, fs=fs)
dvc/utils/serialize/_py.py
+6 -9
View File
@@ -23,8 +23,7 @@ def parse_py(text, path):
with reraise(SyntaxError, PythonFileCorruptedError(path)):
tree = ast.parse(text, filename=path)
result = _ast_tree_to_dict(tree)
return result
return _ast_tree_to_dict(tree)
def parse_py_for_update(text, path):
@@ -67,7 +66,7 @@ def _dump(data, stream):
try:
ast.parse(new_text)
except SyntaxError:
raise PythonFileCorruptedError(
raise PythonFileCorruptedError( # noqa: B904
stream.name,
"Python file structure is corrupted after update params",
)
@@ -161,9 +160,7 @@ def _ast_assign_to_dict(assign, only_self_params=False, lineno=False):
def _get_ast_name(target, only_self_params=False):
if hasattr(target, "id") and not only_self_params:
result = target.id
elif hasattr(target, "attr") and target.value.id == "self":
result = target.attr
else:
raise AttributeError
return result
return target.id
if hasattr(target, "attr") and target.value.id == "self":
return target.attr
raise AttributeError
dvc/utils/serialize/_yaml.py
+1 -1
View File
@@ -26,7 +26,7 @@ def parse_yaml(text, path, typ="safe"):
yaml = YAML(typ=typ)
with reraise(_YAMLError, YAMLFileCorruptedError(path)):
return yaml.load(text) or {}
return yaml.load(text) or {} # noqa: S506
def parse_yaml_for_update(text, path):
dvc/utils/strictyaml.py
+1 -1
View File
@@ -286,7 +286,7 @@ def load(
rev = getattr(fs, "rev", None)
try:
with open_fn(path, encoding=encoding) as fd: # type: ignore
with open_fn(path, encoding=encoding) as fd:
text = fd.read()
data = parse_yaml(text, path, typ="rt" if round_trip else "safe")
except UnicodeDecodeError as exc:
pyproject.toml
+22
View File
@@ -159,6 +159,28 @@ profile = "black"
known_first_party = ["dvc", "dvc_data", "dvc_objects", "dvc_render", "dvc_task", "tests"]
line_length = 79
[tool.ruff]
line-length = 79
ignore = ["S101", "RUF100", "SIM105", "SIM108", "SIM110", "SIM111", "SIM117", "A003", "ARG002"]
select = ["F", "E", "W", "C90", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "ISC", "ICN", "T20", "PT003", "PT008", "PT009", "PT010", "PT015", "PT016", "PT017", "PT020", "PT021", "PT024", "PT025", "PT026", "Q", "RET501", "RET504", "SIM", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "PIE", "RUF"]
target-version = "py38"
[tool.ruff.per-file-ignores]
"scripts/**/*.py" = ["T201"]
[tool.ruff.mccabe]
max-complexity = 15
[tool.ruff.flake8-pytest-style]
fixture-parentheses = false
mark-parentheses = false
parametrize-names-type = "csv"
[tool.ruff.flake8-unused-arguments]
ignore-variadic-names = true
[tool.pytest.ini_options]
log_level = "debug"
addopts = "-ra --cov-config pyproject.toml"
scripts/build.py
+4 -3
View File
@@ -1,5 +1,6 @@
import argparse
import pathlib
import sys
from subprocess import STDOUT, check_call
path = pathlib.Path(__file__).parent.absolute()
@@ -37,7 +38,7 @@ if args.sign_application:
raise NotImplementedError
if not args.application_id:
print("--sign-application requires --application-id")
exit(1)
sys.exit(1)
check_call(
["python", "sign.py", "--application-id", args.application_id],
cwd=pyinstaller,
@@ -64,7 +65,7 @@ if args.sign_installer:
[args.installer_id, args.apple_id_username, args.apple_id_password]
):
print("--sign-installer requires --installer-id")
exit(1)
sys.exit(1)
check_call(
["python", "sign.py", "--installer-id", args.installer_id],
cwd=fpm,
@@ -78,7 +79,7 @@ if args.notarize:
print(
"--notarize requires --apple-id-username and --apple-id-password"
)
exit(1)
sys.exit(1)
check_call(
[
"python",
scripts/fpm/notarize.py
+2 -2
View File
@@ -36,11 +36,11 @@ else:
pkgs = list(path.glob("*.pkg"))
if not pkgs:
print("No pkgs found")
exit(1)
sys.exit(1)
if len(pkgs) > 1:
print("Too many packages")
exit(1)
sys.exit(1)
(pkg,) = pkgs
scripts/fpm/sign.py
+2 -2
View File
@@ -28,11 +28,11 @@ else:
pkgs = list(path.glob("*.pkg"))
if not pkgs:
print("No pkgs found")
exit(1)
sys.exit(1)
if len(pkgs) > 1:
print("Too many packages")
exit(1)
sys.exit(1)
(pkg,) = pkgs
tests/func/experiments/conftest.py
+1 -1
View File
@@ -24,7 +24,7 @@ def http_auth_patch(mocker):
@pytest.fixture(params=[True, False])
def workspace(request, test_queue) -> bool: # noqa
def workspace(request, test_queue) -> bool: # noqa: F811
return request.param
tests/func/experiments/test_show.py
+3 -6
View File
@@ -998,8 +998,7 @@ def test_show_queued_error(tmp_dir, scm, dvc, exp_stage, mocker):
def resolve_commit(rev):
if rev == exp_rev_3:
raise SCMError
else:
return commit_2
return commit_2
mocker.patch.object(
scm,
@@ -1027,8 +1026,7 @@ def test_show_completed_error(tmp_dir, scm, dvc, exp_stage, mocker):
def resolve_commit(rev):
if rev == exp_rev_3:
raise SCMError
else:
return commit_2
return commit_2
mocker.patch.object(
scm,
@@ -1088,8 +1086,7 @@ def test_show_baseline_error(tmp_dir, scm, dvc, exp_stage, mocker):
def resolve_commit(rev):
if rev == baseline_rev:
raise SCMError
else:
return commit_2
return commit_2
mocker.patch.object(
scm,
tests/func/test_add.py
+2 -2
View File
@@ -400,7 +400,7 @@ def test_should_update_state_entry_for_directory_after_add(
assert ret == 0
assert file_md5_counter.mock.call_count == 8
os.rename("data", "data" + ".back")
os.rename("data", "data.back")
ret = main(["checkout"])
assert ret == 0
assert file_md5_counter.mock.call_count == 8
@@ -417,7 +417,7 @@ def test_add_commit(tmp_dir, dvc):
assert os.path.isfile("foo")
assert not os.path.exists(dvc.odb.local.path)
ret = main(["commit", "foo" + ".dvc"])
ret = main(["commit", "foo.dvc"])
assert ret == 0
assert os.path.isfile("foo")
assert dvc.odb.local.exists("acbd18db4cc2f85cedef654fccc4a4d8")
tests/func/test_checkout.py
+12 -20
View File
@@ -12,7 +12,6 @@ from dvc.exceptions import (
CheckoutError,
CheckoutErrorSuggestGit,
ConfirmRemoveError,
DvcException,
NoOutputOrStageError,
)
from dvc.fs import LocalFileSystem, system
@@ -310,27 +309,20 @@ def test_checkout_hook(mocker, tmp_dir, dvc):
def test_checkout_suggest_git(tmp_dir, dvc, scm):
# pylint: disable=no-member
tmp_dir.dvc_gen("foo", "foo")
try:
with pytest.raises(CheckoutErrorSuggestGit) as e:
dvc.checkout(targets="gitbranch")
except DvcException as exc:
assert isinstance(exc, CheckoutErrorSuggestGit)
assert isinstance(exc.__cause__, NoOutputOrStageError)
assert isinstance(exc.__cause__.__cause__, StageFileDoesNotExistError)
assert isinstance(e.value.__cause__, NoOutputOrStageError)
assert isinstance(e.value.__cause__.__cause__, StageFileDoesNotExistError)
try:
dvc.checkout(targets="foo")
except DvcException as exc:
assert isinstance(exc, CheckoutErrorSuggestGit)
assert isinstance(exc.__cause__, NoOutputOrStageError)
assert exc.__cause__.__cause__ is None
with pytest.raises(CheckoutErrorSuggestGit) as e:
dvc.checkout(targets="foobar")
assert isinstance(e.value.__cause__, NoOutputOrStageError)
assert isinstance(e.value.__cause__.__cause__, StageFileDoesNotExistError)
try:
with pytest.raises(CheckoutErrorSuggestGit) as e:
dvc.checkout(targets="looks-like-dvcfile.dvc")
except DvcException as exc:
assert isinstance(exc, CheckoutErrorSuggestGit)
assert isinstance(exc.__cause__, StageFileDoesNotExistError)
assert exc.__cause__.__cause__ is None
assert isinstance(e.value.__cause__, StageFileDoesNotExistError)
assert e.value.__cause__.__cause__ is None
def test_checkout_target_recursive_should_not_remove_other_used_files(
@@ -347,7 +339,7 @@ def test_checkout_recursive_not_directory(tmp_dir, dvc):
ret = main(["add", "foo"])
assert ret == 0
stats = dvc.checkout(targets=["foo" + ".dvc"], recursive=True)
stats = dvc.checkout(targets=["foo.dvc"], recursive=True)
assert stats == {"added": [], "modified": [], "deleted": []}
@@ -603,7 +595,7 @@ def test_stats_does_not_show_changes_by_default(tmp_dir, dvc, scm, capsys):
assert main(["checkout", "--summary"]) == 0
out, _ = capsys.readouterr()
assert "2 files deleted" == out.rstrip()
assert out.rstrip() == "2 files deleted"
@pytest.mark.parametrize("link", ["hardlink", "symlink", "copy"])
tests/func/test_data_cloud.py
-1
View File
@@ -312,7 +312,6 @@ def clean(outs, dvc=None):
if dvc:
dvc.odb.local.clear()
for path in outs:
print(path)
remove(path)
if dvc:
clean_repos()
tests/func/test_ignore.py
+1 -1
View File
@@ -264,7 +264,7 @@ def test_ignore_blank_line(tmp_dir, dvc):
# It is not possible to re-include a file if a parent directory of
# that file is excluded.
# Git doesn’t list excluded directories for performance reasons,
# Git doesn't list excluded directories for performance reasons,
# so any patterns on contained files have no effect,
# no matter where they are defined.
@pytest.mark.parametrize(
tests/func/test_init.py
+1 -1
View File
@@ -65,7 +65,7 @@ def test_init_quiet_should_not_display_welcome_screen(tmp_dir, scm, caplog):
ret = main(["init", "--quiet"])
assert ret == 0
assert "" == caplog.text
assert caplog.text == ""
def test_allow_init_dvc_subdir(tmp_dir, scm, monkeypatch):
tests/func/test_move.py
+1 -2
View File
@@ -29,7 +29,7 @@ def test_move_directory(tmp_dir, dvc):
def test_cmd_move(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
assert main(["move", "foo", "foo" + "1"]) == 0
assert main(["move", "foo", "foo1"]) == 0
assert main(["move", "non-existing-file", "dst"]) != 0
@@ -188,7 +188,6 @@ def test_move_meta(tmp_dir, dvc):
dvc.move("foo", "bar")
res = (tmp_dir / "bar.dvc").read_text()
print(res)
assert res == textwrap.dedent(
"""\
outs:
tests/func/test_odb.py
+1 -1
View File
@@ -187,7 +187,7 @@ def test_shared_cache(tmp_dir, dvc, group):
assert not os.path.exists(cache_dir)
tmp_dir.dvc_gen(
{"file": "file content", "dir": {"file2": "file 2 " "content"}}
{"file": "file content", "dir": {"file2": "file 2 content"}}
)
file_mode = oct(0o444)
tests/func/test_run_multistage.py
+2 -2
View File
@@ -106,7 +106,7 @@ def test_multistage_with_wdir(tmp_dir, dvc):
)
data, _ = Dvcfile(dvc, stage.path)._load()
assert "dir" == data["stages"]["copy-foo1-foo2"]["wdir"]
assert data["stages"]["copy-foo1-foo2"]["wdir"] == "dir"
def test_multistage_always_changed(tmp_dir, dvc):
@@ -305,7 +305,7 @@ def test_run_params_no_exec(tmp_dir, dvc):
def test_run_without_cmd(tmp_dir, dvc, kwargs):
with pytest.raises(InvalidArgumentError) as exc:
dvc.run(**kwargs)
assert "command is not specified" == str(exc.value)
assert str(exc.value) == "command is not specified"
def test_run_overwrite_order(tmp_dir, dvc, run_copy):
tests/func/test_update.py
+2 -2
View File
@@ -283,7 +283,7 @@ def test_update_rev(tmp_dir, dvc, scm, git_dir):
"rev_lock": branch1_head,
}
with open(tmp_dir / "foo", encoding="utf-8") as f:
assert "foobar" == f.read()
assert f.read() == "foobar"
stage = dvc.update(["foo.dvc"], rev="branch2")[0]
assert stage.deps[0].def_repo == {
@@ -292,7 +292,7 @@ def test_update_rev(tmp_dir, dvc, scm, git_dir):
"rev_lock": branch2_head,
}
with open(tmp_dir / "foo", encoding="utf-8") as f:
assert "foobar foo" == f.read()
assert f.read() == "foobar foo"
def test_update_recursive(tmp_dir, dvc, erepo_dir):
tests/pylint_plugin_disable.py
+2 -4
View File
@@ -12,7 +12,7 @@ from pylint.checkers.base import NameChecker
from pylint.checkers.classes import ClassChecker
if TYPE_CHECKING:
from astroid import node_classes, scoped_nodes # noqa
from astroid import node_classes, scoped_nodes
from pylint.lint import PyLinter
@@ -34,14 +34,12 @@ def is_node_in_tests(node: "node_classes.NodeNG"):
return module.file.startswith(TESTS_FOLDER)
def register(linter: "PyLinter"): # noqa
def register(linter: "PyLinter"):
try:
from pylint_plugin_utils import suppress_message
except ImportError:
print("Cannot suppress message. 'pylint_plugin_utils' not installed.")
return
print("Registered custom plugin. Some checks will be disabled for tests.")
for msg, checks in SUPPRESS_CHECKS.items():
for checker_method in checks:
suppress_message(linter, checker_method, msg, is_node_in_tests)
tests/ruff.toml
+3
View File
@@ -0,0 +1,3 @@
extend = "../pyproject.toml"
# keep in sync with `../pyproject.toml`
ignore = ["S", "RUF001", "RUF100", "SIM105", "SIM108", "SIM110", "SIM111", "SIM117", "A003", "ARG001", "ARG002"]
tests/unit/command/test_machine.py
-1
View File
@@ -174,5 +174,4 @@ def test_help_message(tmp_dir, scm, dvc, cmd, use_config, capsys):
except SystemExit:
pass
cap = capsys.readouterr()
print(cap.out)
assert ("--global" in cap.out) is use_config
tests/unit/output/test_local.py
+1 -1
View File
@@ -18,7 +18,7 @@ def test_str_workdir_inside_repo(dvc):
stage = Stage(dvc)
output = Output(stage, "path", cache=False)
assert "path" == str(output)
assert str(output) == "path"
stage = Stage(dvc, wdir="some_folder")
output = Output(stage, "path", cache=False)
tests/unit/proc/test_manager.py
+1 -2
View File
@@ -63,8 +63,7 @@ def test_dead_process(tmp_dir, mocker, running_process):
err = OSError()
err.winerror = 87
raise err
else:
raise ProcessLookupError()
raise ProcessLookupError()
mocker.patch("os.kill", side_effect=side_effect)
with pytest.raises(ProcessLookupError):
tests/unit/stage/test_stage.py
+1 -1
View File
@@ -78,7 +78,7 @@ def test_stage_update(dvc, mocker):
@pytest.mark.skipif(
not isinstance(
threading.current_thread(),
threading._MainThread, # noqa, pylint: disable=protected-access
threading._MainThread, # pylint: disable=protected-access
),
reason="Not running in the main thread.",
)

Some files were not shown because too many files have changed in this diff Show More