update black and change line-length to 88 (#8953)

This commit is contained in:
skshetry
2023-02-04 05:28:07 +02:00
committed by GitHub
parent 5d2701341b
commit a86470ccbc
317 changed files with 1520 additions and 3738 deletions
+1 -1
View File
@@ -11,7 +11,7 @@ ignore=
P1
# use `!r` instead of surrounding by quotes
B028
max_line_length = 79
max_line_length = 88
max-complexity = 10
select = B,C,E,F,W,T4,B902,T,P,TYP
show_source = true
+1 -1
View File
@@ -35,7 +35,7 @@ repos:
args:
- --py38-plus
- repo: https://github.com/psf/black
rev: 22.12.0
rev: 23.1.0
hooks:
- id: black
- repo: https://github.com/PyCQA/isort
+6 -18
View File
@@ -16,9 +16,7 @@ def viztracer_profile(
try:
import viztracer # pylint: disable=import-error
except ImportError:
print( # noqa: T201
"Failed to run profiler, viztracer is not installed"
)
print("Failed to run profiler, viztracer is not installed") # noqa: T201
yield
return
@@ -80,9 +78,7 @@ def instrument(html_output=False):
try:
from pyinstrument import Profiler # pylint: disable=import-error
except ImportError:
print( # noqa: T201
"Failed to run profiler, pyinstrument is not installed"
)
print("Failed to run profiler, pyinstrument is not installed") # noqa: T201
yield
return
@@ -194,9 +190,7 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs):
path_func = _get_path_func("viztracer", "json")
depth = kw.get("viztracer_depth") or -1
log_async = kw.get("viztracer_async") or False
prof = viztracer_profile(
path=path_func, depth=depth, log_async=log_async
)
prof = viztracer_profile(path=path_func, depth=depth, log_async=log_async)
stack.enter_context(prof)
yield
@@ -204,12 +198,8 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs):
def add_debugging_flags(parser):
from argparse import SUPPRESS
parser.add_argument(
"--cprofile", action="store_true", default=False, help=SUPPRESS
)
parser.add_argument(
"--yappi", action="store_true", default=False, help=SUPPRESS
)
parser.add_argument("--cprofile", action="store_true", default=False, help=SUPPRESS)
parser.add_argument("--yappi", action="store_true", default=False, help=SUPPRESS)
parser.add_argument(
"--yappi-separate-threads",
action="store_true",
@@ -224,9 +214,7 @@ def add_debugging_flags(parser):
"--viztracer-async", action="store_true", default=False, help=SUPPRESS
)
parser.add_argument("--cprofile-dump", help=SUPPRESS)
parser.add_argument(
"--pdb", action="store_true", default=False, help=SUPPRESS
)
parser.add_argument("--pdb", action="store_true", default=False, help=SUPPRESS)
parser.add_argument(
"--instrument", action="store_true", default=False, help=SUPPRESS
)
+2 -6
View File
@@ -34,9 +34,7 @@ def get_url(path, repo=None, rev=None, remote=None):
class _OpenContextManager(GCM):
def __init__(
self, func, args, kwds
): # pylint: disable=super-init-not-called
def __init__(self, func, args, kwds): # pylint: disable=super-init-not-called
self.gen = func(*args, **kwds)
self.func, self.args, self.kwds = ( # type: ignore[assignment]
func,
@@ -45,9 +43,7 @@ class _OpenContextManager(GCM):
)
def __getattr__(self, name):
raise AttributeError(
"dvc.api.open() should be used in a with statement."
)
raise AttributeError("dvc.api.open() should be used in a with statement.")
def open( # noqa, pylint: disable=redefined-builtin
+9 -10
View File
@@ -56,8 +56,10 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
if exc.errno == errno.EMFILE:
logger.exception(
(
"too many open files, please visit "
"%s to see how to handle this problem",
"%s to see how to handle this problem"
),
error_link("many-files"),
extra={"tb_only": True},
)
@@ -80,7 +82,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
if cmd:
link = format_link("https://dvc.org/doc/install")
hint = (
f"To install dvc with those dependencies, run:\n"
"To install dvc with those dependencies, run:\n"
"\n"
f"\t{cmd}\n"
"\n"
@@ -91,8 +93,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
hint = f"\nPlease report this bug to {link}. Thank you!"
logger.exception(
"URL '%s' is supported but requires these missing "
"dependencies: %s. %s",
"URL '%s' is supported but requires these missing dependencies: %s. %s",
exc.url,
exc.missing_deps,
hint,
@@ -118,9 +119,11 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
directory = relpath(exc.directory)
logger.exception(
(
"Could not open pickled '%s' cache.\n"
"Remove the '%s' directory and then retry this command."
"\nSee %s for more information.",
"\nSee %s for more information."
),
exc.type,
directory,
error_link("pickle"),
@@ -181,11 +184,7 @@ def main(argv=None): # noqa: C901
set_loggers_level(level)
if level and level <= logging.DEBUG:
from platform import (
platform,
python_implementation,
python_version,
)
from platform import platform, python_implementation, python_version
from dvc import __version__
from dvc.utils.pkg import PKG
+2 -6
View File
@@ -2,9 +2,7 @@ from argparse import _AppendAction
class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
def __call__(
self, parser, namespace, values, option_string=None # noqa: ARG002
):
def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002
from funcy import ldistinct
items = getattr(namespace, self.dest) or []
@@ -13,9 +11,7 @@ class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
class KeyValueArgs(_AppendAction):
def __call__(
self, parser, namespace, values, option_string=None # noqa: ARG002
):
def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002
items = getattr(namespace, self.dest) or {}
for value in filter(bool, values):
key, _, value = value.partition("=")
+1 -3
View File
@@ -28,7 +28,5 @@ def hide_subparsers_from_help(subparsers):
# Need to set `add_help=False`, but avoid setting `help`
# (not even to `argparse.SUPPPRESS`).
# NOTE: The argument is the parent subparser, not the subcommand parser.
cmds = [
cmd for cmd, parser in subparsers.choices.items() if parser.add_help
]
cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help]
subparsers.metavar = "{{{}}}".format(",".join(cmds))
+1 -4
View File
@@ -41,10 +41,7 @@ def _add_annotating_args(parser: argparse.ArgumentParser) -> None:
class CmdAdd(CmdBase):
def run(self):
from dvc.exceptions import (
DvcException,
RecursiveAddingWhileUsingFilename,
)
from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
try:
if len(self.args.targets) > 1 and self.args.file:
+4 -2
View File
@@ -68,10 +68,12 @@ def add_parser(subparsers, parent_parser):
)
cache_dir_parser.add_argument(
"value",
help="Path to cache directory. Relative paths are resolved relative "
help=(
"Path to cache directory. Relative paths are resolved relative "
"to the current directory and saved to config relative to the "
"config file location. If no path is provided, it returns the "
"current cache directory.",
"current cache directory."
),
nargs="?",
).complete = completion.DIR
cache_dir_parser.set_defaults(func=CmdCacheDir)
+10 -10
View File
@@ -62,9 +62,7 @@ class CmdCheckIgnore(CmdBase):
raise DvcException("cannot have both `targets` and `--stdin`")
if self.args.non_matching and not self.args.details:
raise DvcException(
"`--non-matching` is only valid with `--details`"
)
raise DvcException("`--non-matching` is only valid with `--details`")
if self.args.all and not self.args.details:
raise DvcException("`--all` is only valid with `--details`")
@@ -80,9 +78,7 @@ class CmdCheckIgnore(CmdBase):
def add_parser(subparsers, parent_parser):
ADD_HELP = (
"Check whether files or directories are excluded due to `.dvcignore`."
)
ADD_HELP = "Check whether files or directories are excluded due to `.dvcignore`."
parser = subparsers.add_parser(
"check-ignore",
@@ -103,16 +99,20 @@ def add_parser(subparsers, parent_parser):
"--all",
action="store_true",
default=False,
help="Include the target paths which don't match any pattern "
"in the `--details` list.",
help=(
"Include the target paths which don't match any pattern "
"in the `--details` list."
),
)
parser.add_argument(
"-n",
"--non-matching",
action="store_true",
default=False,
help="Include the target paths which don't match any pattern "
"in the `--details` list.",
help=(
"Include the target paths which don't match any pattern "
"in the `--details` list."
),
)
parser.add_argument(
"--stdin",
+2 -6
View File
@@ -22,9 +22,7 @@ def log_changes(stats):
continue
for entry in entries:
ui.write(
f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t"
)
ui.write(f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t")
class CmdCheckout(CmdBase):
@@ -46,9 +44,7 @@ class CmdCheckout(CmdBase):
if self.args.summary:
default_message = "No changes."
msg = get_summary(
sorted(stats.items(), key=operator.itemgetter(0))
)
msg = get_summary(sorted(stats.items(), key=operator.itemgetter(0)))
ui.write(msg or default_message)
else:
log_changes(stats)
+1 -3
View File
@@ -24,9 +24,7 @@ class CmdCommit(CmdBase):
force=self.args.force,
)
except DvcException:
logger.exception(
"failed to commit%s", (" " + target) if target else ""
)
logger.exception("failed to commit%s", (" " + target) if target else "")
return 1
return 0
+3 -9
View File
@@ -17,9 +17,7 @@ def _name_type(value):
match = re.match(NAME_REGEX, value)
if not match:
raise argparse.ArgumentTypeError(
"name argument should look like "
"remote.name.option or "
"section.option"
"name argument should look like remote.name.option or section.option"
)
return (
bool(match.group("remote")),
@@ -70,9 +68,7 @@ class CmdConfig(CmdBaseNoRepo):
for level in levels:
conf = self.config.read(level)
prefix = self._config_file_prefix(
self.args.show_origin, self.config, level
)
prefix = self._config_file_prefix(self.args.show_origin, self.config, level)
configs = list(self._format_config(conf, prefix))
if configs:
ui.write("\n".join(configs))
@@ -131,9 +127,7 @@ class CmdConfig(CmdBaseNoRepo):
raise ConfigError(f"{name} '{section}' doesn't exist")
if opt and opt not in conf[section]:
raise ConfigError(
f"option '{opt}' doesn't exist in {name} '{section}'"
)
raise ConfigError(f"option '{opt}' doesn't exist in {name} '{section}'")
def _get_appropriate_levels(self, levels):
if levels:
+5 -5
View File
@@ -86,9 +86,7 @@ def _collect_targets(repo, target, outs):
targets.extend([str(out) for out in stage.outs])
continue
for out in outs_trie.itervalues( # noqa: B301
prefix=repo.fs.path.parts(path)
):
for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)): # noqa: B301
targets.extend(str(out))
return targets
@@ -204,7 +202,9 @@ def add_parser(subparsers, parent_parser):
dag_parser.add_argument(
"target",
nargs="?",
help="Stage name or output to show pipeline for. "
"Finds all stages in the workspace by default.",
help=(
"Stage name or output to show pipeline for. "
"Finds all stages in the workspace by default."
),
)
dag_parser.set_defaults(func=CmdDAG)
+9 -18
View File
@@ -8,11 +8,7 @@ from funcy import chunks, compact, log_durations
from dvc.cli import completion
from dvc.cli.actions import CommaSeparatedArgs
from dvc.cli.command import CmdBase
from dvc.cli.utils import (
append_doc_link,
fix_subparsers,
hide_subparsers_from_help,
)
from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help
from dvc.ui import ui
from dvc.utils import colorize
@@ -39,20 +35,19 @@ class CmdDataStatus(CmdBase):
}
HINTS = {
"not_in_cache": ('use "dvc fetch <file>..." to download files',),
"committed": (
"git commit the corresponding dvc files to update the repo",
),
"committed": ("git commit the corresponding dvc files to update the repo",),
"uncommitted": (
'use "dvc commit <file>..." to track changes',
'use "dvc checkout <file>..." to discard changes',
),
"untracked": (
(
'use "git add <file> ..." or '
'"dvc add <file>..." to commit to git or to dvc',
'"dvc add <file>..." to commit to git or to dvc'
),
),
"git_dirty": (
"there are {}changes not tracked by dvc, "
'use "git status" to see',
'there are {}changes not tracked by dvc, use "git status" to see',
),
}
@@ -96,8 +91,7 @@ class CmdDataStatus(CmdBase):
if isinstance(stage_status, dict):
items = [
": ".join([state, file])
for file, state in stage_status.items()
": ".join([state, file]) for file, state in stage_status.items()
]
else:
items = stage_status
@@ -180,9 +174,7 @@ class CmdDataLs(CmdBase):
filter_labels = set(self.args.labels)
filter_types = set(self.args.type)
d = ls(
self.repo, targets=self.args.targets, recursive=self.args.recursive
)
d = ls(self.repo, targets=self.args.targets, recursive=self.args.recursive)
self._show_table(
d,
filter_labels=filter_labels,
@@ -205,8 +197,7 @@ def add_parser(subparsers, parent_parser):
fix_subparsers(data_subparsers)
DATA_STATUS_HELP = (
"Show changes between the last git commit, "
"the dvcfiles and the workspace."
"Show changes between the last git commit, the dvcfiles and the workspace."
)
data_status_parser = data_subparsers.add_parser(
"status",
+13 -12
View File
@@ -262,9 +262,7 @@ def add_parser(subparsers, _parent_parser):
push_parser.set_defaults(func=CmdDataPush)
# Fetch
FETCH_HELP = (
"Download files or directories from remote storage to the cache."
)
FETCH_HELP = "Download files or directories from remote storage to the cache."
fetch_parser = subparsers.add_parser(
"fetch",
@@ -320,9 +318,7 @@ def add_parser(subparsers, _parent_parser):
fetch_parser.set_defaults(func=CmdDataFetch)
# Status
STATUS_HELP = (
"Show changed stages, compare local cache and a remote storage."
)
STATUS_HELP = "Show changed stages, compare local cache and a remote storage."
status_parser = subparsers.add_parser(
"status",
@@ -360,24 +356,29 @@ def add_parser(subparsers, _parent_parser):
"--all-branches",
action="store_true",
default=False,
help="Show status of a local cache compared to a remote repository "
"for all branches.",
help=(
"Show status of a local cache compared to a remote repository "
"for all branches."
),
)
status_parser.add_argument(
"-T",
"--all-tags",
action="store_true",
default=False,
help="Show status of a local cache compared to a remote repository "
"for all tags.",
help=(
"Show status of a local cache compared to a remote repository for all tags."
),
)
status_parser.add_argument(
"-A",
"--all-commits",
action="store_true",
default=False,
help="Show status of a local cache compared to a remote repository "
"for all commits.",
help=(
"Show status of a local cache compared to a remote repository "
"for all commits."
),
)
status_parser.add_argument(
"-d",
+8 -13
View File
@@ -112,9 +112,7 @@ class CmdDiff(CmdBase):
return
states_summary = ", ".join(
f"{summary[state]} {state}"
for state in states
if summary[state] > 0
f"{summary[state]} {state}" for state in states if summary[state] > 0
)
ui.write("files summary:", states_summary)
@@ -122,9 +120,7 @@ class CmdDiff(CmdBase):
from dvc.exceptions import DvcException
try:
diff = self.repo.diff(
self.args.a_rev, self.args.b_rev, self.args.targets
)
diff = self.repo.diff(self.args.a_rev, self.args.b_rev, self.args.targets)
show_hash = self.args.show_hash
hide_missing = self.args.b_rev or self.args.hide_missing
if hide_missing:
@@ -133,9 +129,11 @@ class CmdDiff(CmdBase):
for key, entries in diff.items():
entries = sorted(
entries,
key=lambda entry: entry["path"]["old"]
key=lambda entry: (
entry["path"]["old"]
if isinstance(entry["path"], dict)
else entry["path"],
else entry["path"]
),
)
if not show_hash:
for entry in entries:
@@ -170,10 +168,7 @@ def add_parser(subparsers, parent_parser):
diff_parser.add_argument(
"--targets",
nargs="*",
help=(
"Specific DVC-tracked files to compare. "
"Accepts one or more file paths."
),
help="Specific DVC-tracked files to compare. Accepts one or more file paths.",
metavar="<paths>",
).complete = completion.FILE
diff_parser.add_argument(
@@ -184,7 +179,7 @@ def add_parser(subparsers, parent_parser):
)
diff_parser.add_argument(
"b_rev",
help=("New Git commit to compare (defaults to the current workspace)"),
help="New Git commit to compare (defaults to the current workspace)",
nargs="?",
)
diff_parser.add_argument(
+2 -7
View File
@@ -1,10 +1,6 @@
import argparse
from dvc.cli.utils import (
append_doc_link,
fix_subparsers,
hide_subparsers_from_help,
)
from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help
from dvc.commands.experiments import (
apply,
branch,
@@ -54,8 +50,7 @@ def add_parser(subparsers, parent_parser):
experiments_subparsers = experiments_parser.add_subparsers(
dest="cmd",
help="Use `dvc experiments CMD --help` to display "
"command-specific help.",
help="Use `dvc experiments CMD --help` to display command-specific help.",
)
fix_subparsers(experiments_subparsers)
+2 -8
View File
@@ -10,19 +10,13 @@ logger = logging.getLogger(__name__)
class CmdExperimentsApply(CmdBase):
def run(self):
self.repo.experiments.apply(
self.args.experiment, force=self.args.force
)
self.repo.experiments.apply(self.args.experiment, force=self.args.force)
return 0
def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_APPLY_HELP = (
"Apply the changes from an experiment to your workspace."
)
EXPERIMENTS_APPLY_HELP = "Apply the changes from an experiment to your workspace."
experiments_apply_parser = experiments_subparsers.add_parser(
"apply",
parents=[parent_parser],
+1 -5
View File
@@ -9,14 +9,12 @@ logger = logging.getLogger(__name__)
class CmdExperimentsBranch(CmdBase):
def run(self):
self.repo.experiments.branch(self.args.experiment, self.args.branch)
return 0
def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch."
experiments_branch_parser = experiments_subparsers.add_parser(
"branch",
@@ -28,7 +26,5 @@ def add_parser(experiments_subparsers, parent_parser):
experiments_branch_parser.add_argument(
"experiment", help="Experiment to be promoted."
)
experiments_branch_parser.add_argument(
"branch", help="Name for the new Git branch"
)
experiments_branch_parser.add_argument("branch", help="Name for the new Git branch")
experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)
+1 -5
View File
@@ -13,7 +13,6 @@ logger = logging.getLogger(__name__)
class CmdExperimentsDiff(CmdBase):
def run(self):
try:
diff = self.repo.experiments.diff(
a_rev=self.args.a_rev,
@@ -53,10 +52,7 @@ class CmdExperimentsDiff(CmdBase):
def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_DIFF_HELP = (
"Show changes between experiments in the DVC repository."
)
EXPERIMENTS_DIFF_HELP = "Show changes between experiments in the DVC repository."
experiments_diff_parser = experiments_subparsers.add_parser(
"diff",
+1 -4
View File
@@ -9,10 +9,7 @@ class CmdExecutorRun(CmdBaseNoRepo):
"""Run an experiment executor."""
def run(self):
from dvc.repo.experiments.executor.base import (
BaseExecutor,
ExecutorInfo,
)
from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo
from dvc.utils.serialize import load_json
info = ExecutorInfo.from_dict(load_json(self.args.infofile))
+2 -7
View File
@@ -41,10 +41,7 @@ class CmdExperimentsGC(CmdBase):
msg += " and all git tags"
if self.args.commit_date:
msg += (
" and all git commits before date "
f"{self.args.commit_date}"
)
msg += f" and all git commits before date {self.args.commit_date}"
msg += " of the current repo."
if self.args.queued:
@@ -79,11 +76,9 @@ class CmdExperimentsGC(CmdBase):
def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
EXPERIMENTS_GC_DESCRIPTION = (
"Removes all experiments which are not derived from the specified"
"Git revisions."
"Removes all experiments which are not derived from the specifiedGit revisions."
)
experiments_gc_parser = experiments_subparsers.add_parser(
"gc",
+25 -16
View File
@@ -92,9 +92,7 @@ class CmdExperimentsInit(CmdBase):
if new_out_dirs:
out_dirs_paths = humanize.join(map(path_fmt, new_out_dirs))
ui.write(
f"Creating output directories: {out_dirs_paths}", styled=True
)
ui.write(f"Creating output directories: {out_dirs_paths}", styled=True)
ui.write(
f"Creating [b]{self.args.name}[/b] stage in [green]dvc.yaml[/]",
@@ -120,7 +118,6 @@ class CmdExperimentsInit(CmdBase):
def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_INIT_HELP = "Quickly setup any project to use experiments."
experiments_init_parser = experiments_subparsers.add_parser(
"init",
@@ -167,35 +164,47 @@ def add_parser(experiments_subparsers, parent_parser):
)
experiments_init_parser.add_argument(
"--code",
help="Path to the source file or directory "
help=(
"Path to the source file or directory "
"which your experiments depend"
f" (default: {CmdExperimentsInit.CODE})",
f" (default: {CmdExperimentsInit.CODE})"
),
)
experiments_init_parser.add_argument(
"--data",
help="Path to the data file or directory "
help=(
"Path to the data file or directory "
"which your experiments depend"
f" (default: {CmdExperimentsInit.DATA})",
f" (default: {CmdExperimentsInit.DATA})"
),
)
experiments_init_parser.add_argument(
"--models",
help="Path to the model file or directory for your experiments"
f" (default: {CmdExperimentsInit.MODELS})",
help=(
"Path to the model file or directory for your experiments"
f" (default: {CmdExperimentsInit.MODELS})"
),
)
experiments_init_parser.add_argument(
"--params",
help="Path to the parameters file for your experiments"
f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})",
help=(
"Path to the parameters file for your experiments"
f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})"
),
)
experiments_init_parser.add_argument(
"--metrics",
help="Path to the metrics file for your experiments"
f" (default: {CmdExperimentsInit.DEFAULT_METRICS})",
help=(
"Path to the metrics file for your experiments"
f" (default: {CmdExperimentsInit.DEFAULT_METRICS})"
),
)
experiments_init_parser.add_argument(
"--plots",
help="Path to the plots file or directory for your experiments"
f" (default: {CmdExperimentsInit.PLOTS})",
help=(
"Path to the plots file or directory for your experiments"
f" (default: {CmdExperimentsInit.PLOTS})"
),
)
experiments_init_parser.add_argument(
"--live",
+3 -9
View File
@@ -11,9 +11,7 @@ logger = logging.getLogger(__name__)
class CmdExperimentsPull(CmdBase):
def raise_error_if_all_disabled(self):
if not any(
[self.args.experiment, self.args.all_commits, self.args.rev]
):
if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
raise InvalidArgumentError(
"Either provide an `experiment` argument, or use the "
"`--rev` or `--all-commits` flag."
@@ -74,10 +72,7 @@ def add_parser(experiments_subparsers, parent_parser):
"--no-cache",
action="store_false",
dest="pull_cache",
help=(
"Do not pull cached outputs for this experiment from DVC remote "
"storage."
),
help="Do not pull cached outputs for this experiment from DVC remote storage.",
)
experiments_pull_parser.add_argument(
"-r",
@@ -92,8 +87,7 @@ def add_parser(experiments_subparsers, parent_parser):
type=int,
metavar="<number>",
help=(
"Number of jobs to run simultaneously when pulling from DVC "
"remote storage."
"Number of jobs to run simultaneously when pulling from DVC remote storage."
),
)
experiments_pull_parser.add_argument(
+6 -13
View File
@@ -12,16 +12,13 @@ logger = logging.getLogger(__name__)
class CmdExperimentsPush(CmdBase):
def raise_error_if_all_disabled(self):
if not any(
[self.args.experiment, self.args.all_commits, self.args.rev]
):
if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
raise InvalidArgumentError(
"Either provide an `experiment` argument, or use the "
"`--rev` or `--all-commits` flag."
)
def run(self):
self.raise_error_if_all_disabled()
pushed_exps = self.repo.experiments.push(
@@ -47,8 +44,10 @@ class CmdExperimentsPush(CmdBase):
if not self.args.push_cache:
ui.write(
"To push cached outputs",
(
"for this experiment to DVC remote storage,"
"re-run this command without '--no-cache'.",
"re-run this command without '--no-cache'."
),
)
return 0
@@ -76,10 +75,7 @@ def add_parser(experiments_subparsers, parent_parser):
"--no-cache",
action="store_false",
dest="push_cache",
help=(
"Do not push cached outputs for this experiment to DVC remote "
"storage."
),
help="Do not push cached outputs for this experiment to DVC remote storage.",
)
experiments_push_parser.add_argument(
"-r",
@@ -93,10 +89,7 @@ def add_parser(experiments_subparsers, parent_parser):
"--jobs",
type=int,
metavar="<number>",
help=(
"Number of jobs to run simultaneously when pushing to DVC remote "
"storage."
),
help="Number of jobs to run simultaneously when pushing to DVC remote storage.",
)
experiments_push_parser.add_argument(
"--run-cache",
-1
View File
@@ -24,7 +24,6 @@ class CmdExperimentsRemove(CmdBase):
)
def run(self):
self.check_arguments()
removed_list = self.repo.experiments.remove(
+2 -6
View File
@@ -17,13 +17,10 @@ class CmdExperimentsRun(CmdRepro):
if self.args.checkpoint_resume:
if self.args.reset:
raise InvalidArgumentError(
"--reset and --rev are mutually exclusive."
)
raise InvalidArgumentError("--reset and --rev are mutually exclusive.")
if not (self.args.queue or self.args.tmp_dir):
raise InvalidArgumentError(
"--rev can only be used in conjunction with "
"--queue or --temp."
"--rev can only be used in conjunction with --queue or --temp."
)
if self.args.reset:
@@ -51,7 +48,6 @@ class CmdExperimentsRun(CmdRepro):
def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
experiments_run_parser = experiments_subparsers.add_parser(
"run",
-1
View File
@@ -11,7 +11,6 @@ logger = logging.getLogger(__name__)
class CmdExperimentsSave(CmdBase):
def run(self):
try:
ref = self.repo.experiments.save(
name=self.args.name,
+7 -22
View File
@@ -84,9 +84,7 @@ def _collect_rows( # noqa: C901
sort_by, metric_names, param_names
)
reverse = sort_order == "desc"
experiments = _sort_exp(
experiments, sort_path, sort_name, sort_type, reverse
)
experiments = _sort_exp(experiments, sort_path, sort_name, sort_type, reverse)
new_checkpoint = True
for i, (rev, results) in enumerate(experiments.items()):
@@ -116,9 +114,7 @@ def _collect_rows( # noqa: C901
typ = "baseline"
elif tip:
if tip == parent_tip:
typ = (
"checkpoint_tip" if new_checkpoint else "checkpoint_commit"
)
typ = "checkpoint_tip" if new_checkpoint else "checkpoint_commit"
elif parent_rev == base_rev:
typ = "checkpoint_base"
else:
@@ -135,9 +131,7 @@ def _collect_rows( # noqa: C901
row_dict["Experiment"] = exp.get("name", "")
row_dict["rev"] = name_rev
row_dict["typ"] = typ
row_dict["Created"] = format_time(
exp.get("timestamp"), fill_value, iso
)
row_dict["Created"] = format_time(exp.get("timestamp"), fill_value, iso)
row_dict["parent"] = parent
row_dict["State"] = state
row_dict["Executor"] = exp.get("executor", fill_value)
@@ -371,9 +365,7 @@ def show_experiments( # noqa: C901
if not csv:
merge_headers = ["Experiment", "rev", "typ", "parent"]
td.column("Experiment")[:] = map(
prepare_exp_id, td.as_dict(merge_headers)
)
td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers))
td.drop(*merge_headers[1:])
headers = {
@@ -410,9 +402,7 @@ def show_experiments( # noqa: C901
cols_to_drop = set()
if drop is not None:
cols_to_drop = {
col for col in td.keys() if re.match(drop, col) # noqa: SIM118
}
cols_to_drop = {col for col in td.keys() if re.match(drop, col)} # noqa: SIM118
if pcp:
cols_to_drop.add("Created")
td.drop(*cols_to_drop)
@@ -491,9 +481,7 @@ class CmdExperimentsShow(CmdBase):
ui.write_json(all_experiments, default=_format_json)
else:
precision = (
self.args.precision or None
if self.args.csv
else DEFAULT_PRECISION
self.args.precision or None if self.args.csv else DEFAULT_PRECISION
)
fill_value = "" if self.args.csv else FILL_VALUE
iso = self.args.csv
@@ -582,10 +570,7 @@ def add_parser(experiments_subparsers, parent_parser):
)
experiments_show_parser.add_argument(
"--sort-order",
help=(
"Sort order to use with --sort-by."
" Defaults to ascending ('asc')."
),
help="Sort order to use with --sort-by. Defaults to ascending ('asc').",
choices=("asc", "desc"),
default="asc",
)
+6 -9
View File
@@ -40,14 +40,9 @@ class CmdGC(CmdBase):
elif self.args.all_tags:
msg += " and all git tags"
if self.args.commit_date:
msg += (
" and all git commits before date "
f"{self.args.commit_date}"
)
msg += f" and all git commits before date {self.args.commit_date}"
if self.args.rev:
msg += (
f" and last {self.args.num} commits from {self.args.rev}"
)
msg += f" and last {self.args.num} commits from {self.args.rev}"
if self.args.all_experiments:
msg += " and all experiments"
@@ -198,9 +193,11 @@ def add_parser(subparsers, parent_parser):
dest="repos",
type=str,
nargs="*",
help="Keep data files required by these projects "
help=(
"Keep data files required by these projects "
"in addition to the current one. "
"Useful if you share a single cache across repos.",
"Useful if you share a single cache across repos."
),
metavar="<paths>",
)
gc_parser.set_defaults(func=CmdGC)
+5 -5
View File
@@ -16,9 +16,7 @@ class CmdGet(CmdBaseNoRepo):
from dvc.ui import ui
try:
url = get_url(
self.args.path, repo=self.args.url, rev=self.args.rev
)
url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
ui.write(url, force=True)
except DvcException:
logger.exception("failed to show URL")
@@ -85,8 +83,10 @@ def add_parser(subparsers, parent_parser):
get_parser.add_argument(
"--show-url",
action="store_true",
help="Print the storage location (URL) the target data would be "
"downloaded from, and exit.",
help=(
"Print the storage location (URL) the target data would be "
"downloaded from, and exit."
),
)
get_parser.add_argument(
"-j",
+4 -2
View File
@@ -90,8 +90,10 @@ def add_parser(subparsers, parent_parser):
"--no-download",
action="store_true",
default=False,
help="Create .dvc file including target data hash value(s)"
" but do not actually download the file(s).",
help=(
"Create .dvc file including target data hash value(s)"
" but do not actually download the file(s)."
),
)
import_parser.add_argument(
"-j",
+13 -12
View File
@@ -29,8 +29,10 @@ class CmdImportUrl(CmdBase):
)
except DvcException:
logger.exception(
(
"failed to import %s. You could also try downloading "
"it manually, and adding it with `dvc add`.",
"it manually, and adding it with `dvc add`."
),
self.args.url,
)
return 1
@@ -40,9 +42,7 @@ class CmdImportUrl(CmdBase):
def add_parser(subparsers, parent_parser):
from .add import _add_annotating_args
IMPORT_HELP = (
"Download or copy file from URL and take it under DVC control."
)
IMPORT_HELP = "Download or copy file from URL and take it under DVC control."
import_parser = subparsers.add_parser(
"import-url",
@@ -53,7 +53,8 @@ def add_parser(subparsers, parent_parser):
)
import_parser.add_argument(
"url",
help="Location of the data to download. Supported URLs:\n"
help=(
"Location of the data to download. Supported URLs:\n"
"/absolute/path/to/file/or/dir\n"
"relative/path/to/file/or/dir\n"
"C:\\\\path\\to\\file\\or\\dir\n"
@@ -62,7 +63,8 @@ def add_parser(subparsers, parent_parser):
"gs://bucket/path/to/file/or/dir\n"
"hdfs://example.com/path/to/file\n"
"ssh://example.com/absolute/path/to/file/or/dir\n"
"remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
"remote://remote_name/path/to/file/or/dir (see `dvc remote`)"
),
)
import_parser.add_argument(
"out", nargs="?", help="Destination path to put files to."
@@ -95,8 +97,10 @@ def add_parser(subparsers, parent_parser):
"--no-download",
action="store_true",
default=False,
help="Create .dvc file including target data hash value(s)"
" but do not actually download the file(s).",
help=(
"Create .dvc file including target data hash value(s)"
" but do not actually download the file(s)."
),
)
import_parser.add_argument(
"-j",
@@ -112,10 +116,7 @@ def add_parser(subparsers, parent_parser):
"--version-aware",
action="store_true",
default=False,
help=(
"Import using cloud versioning. Implied if the URL contains a "
"version ID."
),
help="Import using cloud versioning. Implied if the URL contains a version ID.",
)
_add_annotating_args(import_parser)
import_parser.set_defaults(func=CmdImportUrl)
+4 -4
View File
@@ -75,8 +75,9 @@ def add_parser(subparsers, parent_parser):
"--no-scm",
action="store_true",
default=False,
help="Initiate DVC in directory that is "
"not tracked by any SCM tool (e.g. Git).",
help=(
"Initiate DVC in directory that is not tracked by any SCM tool (e.g. Git)."
),
)
init_parser.add_argument(
"-f",
@@ -84,8 +85,7 @@ def add_parser(subparsers, parent_parser):
action="store_true",
default=False,
help=(
"Overwrite existing '.dvc/' directory. "
"This operation removes local cache."
"Overwrite existing '.dvc/' directory. This operation removes local cache."
),
)
init_parser.add_argument(
+4 -2
View File
@@ -31,7 +31,9 @@ def add_parser(subparsers, parent_parser):
"--use-pre-commit-tool",
action="store_true",
default=False,
help="Install DVC hooks using pre-commit "
"(https://pre-commit.com) if it is installed.",
help=(
"Install DVC hooks using pre-commit "
"(https://pre-commit.com) if it is installed."
),
)
install_parser.set_defaults(func=CmdInstall)
+9 -28
View File
@@ -18,7 +18,6 @@ class MachineDisabledError(ConfigError):
class CmdMachineConfig(CmdConfig):
def __init__(self, args):
super().__init__(args)
if not self.config["feature"].get("machine", False):
raise MachineDisabledError
@@ -163,14 +162,11 @@ class CmdMachineRename(CmdBase):
all_config = self.config.load_config_to_level(None)
if self.args.new in all_config.get("machine", {}):
raise ConfigError(
"Rename failed. Machine '{}' already exists.".format(
self.args.new
)
f"Rename failed. Machine '{self.args.new}' already exists."
)
ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.")
def run(self):
self._check_before_rename()
with self.config.edit(self.args.level) as conf:
@@ -209,9 +205,7 @@ class CmdMachineDefault(CmdMachineConfig):
if self.args.unset:
conf["core"].pop("machine", None)
else:
merged_conf = self.config.load_config_to_level(
self.args.level
)
merged_conf = self.config.load_config_to_level(self.args.level)
if (
self.args.name in conf["machine"]
or self.args.name in merged_conf["machine"]
@@ -219,8 +213,7 @@ class CmdMachineDefault(CmdMachineConfig):
conf["core"]["machine"] = self.args.name
else:
raise ConfigError(
"default machine must be present in machine "
"list."
"default machine must be present in machine list."
)
return 0
@@ -251,7 +244,6 @@ class CmdMachineStatus(CmdBase):
all_status: List[Dict],
td: TabularData,
):
if not all_status:
row = [
name,
@@ -343,8 +335,7 @@ def add_parser(subparsers, parent_parser):
"cloud",
help="Machine cloud. See full list of supported clouds at {}".format(
format_link(
"https://github.com/iterative/"
"terraform-provider-iterative#machine"
"https://github.com/iterative/terraform-provider-iterative#machine"
)
),
)
@@ -372,9 +363,7 @@ def add_parser(subparsers, parent_parser):
help=machine_DEFAULT_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
machine_default_parser.add_argument(
"name", nargs="?", help="Name of the machine"
)
machine_default_parser.add_argument("name", nargs="?", help="Name of the machine")
machine_default_parser.add_argument(
"-u",
"--unset",
@@ -414,9 +403,7 @@ def add_parser(subparsers, parent_parser):
formatter_class=argparse.RawDescriptionHelpFormatter,
)
machine_modify_parser.add_argument("name", help="Name of the machine")
machine_modify_parser.add_argument(
"option", help="Name of the option to modify."
)
machine_modify_parser.add_argument("option", help="Name of the option to modify.")
machine_modify_parser.add_argument(
"value", nargs="?", help="(optional) Value of the option."
)
@@ -449,9 +436,7 @@ def add_parser(subparsers, parent_parser):
help=machine_REMOVE_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
machine_remove_parser.add_argument(
"name", help="Name of the machine to remove."
)
machine_remove_parser.add_argument("name", help="Name of the machine to remove.")
machine_remove_parser.set_defaults(func=CmdMachineRemove)
machine_CREATE_HELP = "Create and start a machine instance."
@@ -462,14 +447,10 @@ def add_parser(subparsers, parent_parser):
help=machine_CREATE_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
machine_create_parser.add_argument(
"name", help="Name of the machine to create."
)
machine_create_parser.add_argument("name", help="Name of the machine to create.")
machine_create_parser.set_defaults(func=CmdMachineCreate)
machine_STATUS_HELP = (
"List the status of running instances for one/all machines."
)
machine_STATUS_HELP = "List the status of running instances for one/all machines."
machine_status_parser = machine_subparsers.add_parser(
"status",
parents=[parent_parser],
+2 -6
View File
@@ -14,9 +14,7 @@ class CmdMove(CmdBase):
try:
self.repo.move(self.args.src, self.args.dst)
except DvcException:
msg = "failed to move '{}' -> '{}'".format(
self.args.src, self.args.dst
)
msg = f"failed to move '{self.args.src}' -> '{self.args.dst}'"
logger.exception(msg)
return 1
return 0
@@ -40,7 +38,5 @@ def add_parser(subparsers, parent_parser):
move_parser.add_argument(
"src", help="Source path to a data file or directory."
).complete = completion.FILE
move_parser.add_argument(
"dst", help="Destination path."
).complete = completion.FILE
move_parser.add_argument("dst", help="Destination path.").complete = completion.FILE
move_parser.set_defaults(func=CmdMove)
+1 -1
View File
@@ -79,7 +79,7 @@ def add_parser(subparsers, parent_parser):
params_diff_parser.add_argument(
"b_rev",
nargs="?",
help=("New Git commit to compare (defaults to the current workspace)"),
help="New Git commit to compare (defaults to the current workspace)",
)
params_diff_parser.add_argument(
"--targets",
+12 -31
View File
@@ -18,9 +18,7 @@ logger = logging.getLogger(__name__)
def _show_json(renderers, split=False):
from dvc.render.convert import to_json
result = {
renderer.name: to_json(renderer, split) for renderer in renderers
}
result = {renderer.name: to_json(renderer, split) for renderer in renderers}
ui.write_json(result)
@@ -90,10 +88,8 @@ class CmdPlots(CmdBase):
)
if html_template_path and not os.path.isabs(html_template_path):
assert self.repo.dvc_dir
html_template_path = os.path.join(
self.repo.dvc_dir, html_template_path
)
return html_template_path
html_template_path = os.path.join(self.repo.dvc_dir, html_template_path)
return html_template_path # noqa: RET504
def run(self) -> int: # noqa: C901
from pathlib import Path
@@ -106,14 +102,11 @@ class CmdPlots(CmdBase):
logger.error("please specify a target for `--show-vega`")
return 1
if len(self.args.targets) > 1:
logger.error(
"you can only specify one target for `--show-vega`"
)
logger.error("you can only specify one target for `--show-vega`")
return 1
if self.args.json:
logger.error(
"'--show-vega' and '--json' are mutually exclusive "
"options."
"'--show-vega' and '--json' are mutually exclusive options."
)
return 1
@@ -126,17 +119,14 @@ class CmdPlots(CmdBase):
if not plots_data:
ui.error_write(
"No plots were loaded, "
"visualization file will not be created."
"No plots were loaded, visualization file will not be created."
)
out: str = self.args.out or self.repo.config.get("plots", {}).get(
"out_dir", "dvc_plots"
)
renderers_out = (
out if self.args.json else os.path.join(out, "static")
)
renderers_out = out if self.args.json else os.path.join(out, "static")
renderers = match_defs_renderers(
data=plots_data,
@@ -249,8 +239,7 @@ def add_parser(subparsers, parent_parser):
fix_subparsers(plots_subparsers)
SHOW_HELP = (
"Generate plots from target files or from `plots`"
" definitions in `dvc.yaml`."
"Generate plots from target files or from `plots` definitions in `dvc.yaml`."
)
plots_show_parser = plots_subparsers.add_parser(
"show",
@@ -273,8 +262,7 @@ def add_parser(subparsers, parent_parser):
plots_show_parser.set_defaults(func=CmdPlotsShow)
PLOTS_DIFF_HELP = (
"Show multiple versions of a plot by overlaying them "
"in a single image."
"Show multiple versions of a plot by overlaying them in a single image."
)
plots_diff_parser = plots_subparsers.add_parser(
"diff",
@@ -332,9 +320,7 @@ def add_parser(subparsers, parent_parser):
)
plots_modify_parser.set_defaults(func=CmdPlotsModify)
TEMPLATES_HELP = (
"List built-in plots templates or show JSON specification for one."
)
TEMPLATES_HELP = "List built-in plots templates or show JSON specification for one."
plots_templates_parser = plots_subparsers.add_parser(
"templates",
parents=[parent_parser],
@@ -360,11 +346,8 @@ def _add_props_arguments(parser):
"--template",
nargs="?",
default=None,
help=(
"Special JSON or HTML schema file to inject with the data. "
"See {}".format(
help="Special JSON or HTML schema file to inject with the data. See {}".format(
format_link("https://man.dvc.org/plots#plot-templates")
)
),
metavar="<path>",
).complete = completion.FILE
@@ -381,9 +364,7 @@ def _add_props_arguments(parser):
default=None, # Use default None to distinguish when it's not used
help="Provided CSV or TSV datafile does not have a header.",
)
parser.add_argument(
"--title", default=None, metavar="<text>", help="Plot title."
)
parser.add_argument("--title", default=None, metavar="<text>", help="Plot title.")
parser.add_argument(
"--x-label", default=None, help="X axis label", metavar="<text>"
)
+1 -2
View File
@@ -20,8 +20,7 @@ class CmdQueueKill(CmdBase):
def add_parser(queue_subparsers, parent_parser):
QUEUE_KILL_HELP = (
"Gracefully interrupt running experiment queue tasks "
"(equivalent to Ctrl-C)"
"Gracefully interrupt running experiment queue tasks (equivalent to Ctrl-C)"
)
queue_kill_parser = queue_subparsers.add_parser(
"kill",
+1 -3
View File
@@ -34,9 +34,7 @@ def add_parser(queue_subparsers, parent_parser):
queue_logs_parser.add_argument(
"-e",
"--encoding",
help=(
"Text encoding for log output. Defaults to system locale encoding."
),
help="Text encoding for log output. Defaults to system locale encoding.",
metavar="<encoding>",
)
queue_logs_parser.add_argument(
-2
View File
@@ -28,7 +28,6 @@ class CmdQueueRemove(CmdBase):
)
def run(self):
self.check_arguments()
if self.args.all:
@@ -57,7 +56,6 @@ class CmdQueueRemove(CmdBase):
def add_parser(queue_subparsers, parent_parser):
QUEUE_REMOVE_HELP = "Remove queued and completed tasks from the queue."
queue_remove_parser = queue_subparsers.add_parser(
"remove",
+3 -11
View File
@@ -12,20 +12,15 @@ class CmdQueueStart(CmdBase):
"""Start exp queue workers."""
def run(self):
started = self.repo.experiments.celery_queue.start_workers(
self.args.jobs
)
started = self.repo.experiments.celery_queue.start_workers(self.args.jobs)
suffix = "s" if started > 1 else ""
ui.write(
f"Started '{started}' new experiments task queue worker{suffix}."
)
ui.write(f"Started '{started}' new experiments task queue worker{suffix}.")
return 0
def add_parser(queue_subparsers, parent_parser):
QUEUE_START_HELP = "Start the experiments task queue worker."
queue_start_parser = queue_subparsers.add_parser(
"start",
@@ -39,10 +34,7 @@ def add_parser(queue_subparsers, parent_parser):
"--jobs",
type=int,
default=1,
help=(
"Maximum number of concurrent queue workers to start. "
"Defaults to 1."
),
help="Maximum number of concurrent queue workers to start. Defaults to 1.",
metavar="<number>",
)
queue_start_parser.set_defaults(func=CmdQueueStart)
+2 -6
View File
@@ -39,9 +39,7 @@ class CmdQueueStatus(CmdBase):
ui.write()
worker_status = self.repo.experiments.celery_queue.worker_status()
active_count = len(
[name for name, task in worker_status.items() if task]
)
active_count = len([name for name, task in worker_status.items() if task])
idle_count = len(worker_status) - active_count
ui.write(f"Worker status: {active_count} active, {idle_count} idle")
@@ -50,9 +48,7 @@ class CmdQueueStatus(CmdBase):
def add_parser(queue_subparsers, parent_parser):
QUEUE_STATUS_HELP = (
"Show the status of experiments queue tasks and workers."
)
QUEUE_STATUS_HELP = "Show the status of experiments queue tasks and workers."
queue_status_parser = queue_subparsers.add_parser(
"status",
parents=[parent_parser],
-1
View File
@@ -26,7 +26,6 @@ class CmdQueueStop(CmdBase):
def add_parser(queue_subparsers, parent_parser):
QUEUE_STOP_HELP = "Stop all experiments task queue workers."
queue_stop_parser = queue_subparsers.add_parser(
"stop",
+4 -12
View File
@@ -95,9 +95,7 @@ class CmdRemoteDefault(CmdRemote):
if self.args.unset:
conf["core"].pop("remote", None)
else:
merged_conf = self.config.load_config_to_level(
self.args.level
)
merged_conf = self.config.load_config_to_level(self.args.level)
if (
self.args.name in conf["remote"]
or self.args.name in merged_conf["remote"]
@@ -208,9 +206,7 @@ def add_parser(subparsers, parent_parser):
help=REMOTE_DEFAULT_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
remote_default_parser.add_argument(
"name", nargs="?", help="Name of the remote"
)
remote_default_parser.add_argument("name", nargs="?", help="Name of the remote")
remote_default_parser.add_argument(
"-u",
"--unset",
@@ -229,9 +225,7 @@ def add_parser(subparsers, parent_parser):
formatter_class=argparse.RawDescriptionHelpFormatter,
)
remote_modify_parser.add_argument("name", help="Name of the remote")
remote_modify_parser.add_argument(
"option", help="Name of the option to modify."
)
remote_modify_parser.add_argument("option", help="Name of the option to modify.")
remote_modify_parser.add_argument(
"value", nargs="?", help="(optional) Value of the option."
)
@@ -262,9 +256,7 @@ def add_parser(subparsers, parent_parser):
help=REMOTE_REMOVE_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
remote_remove_parser.add_argument(
"name", help="Name of the remote to remove."
)
remote_remove_parser.add_argument("name", help="Name of the remote to remove.")
remote_remove_parser.set_defaults(func=CmdRemoteRemove)
REMOTE_RENAME_HELP = "Rename a DVC remote"
remote_rename_parser = remote_subparsers.add_parser(
+1 -2
View File
@@ -22,8 +22,7 @@ class CmdRemove(CmdBase):
def add_parser(subparsers, parent_parser):
REMOVE_HELP = (
"Remove stages from dvc.yaml and/or"
" stop tracking files or directories."
"Remove stages from dvc.yaml and/or stop tracking files or directories."
)
remove_parser = subparsers.add_parser(
"remove",
+13 -18
View File
@@ -10,9 +10,7 @@ class CmdRepro(CmdBase):
def run(self):
from dvc.ui import ui
stages = self.repo.reproduce(
**self._common_kwargs, **self._repro_kwargs
)
stages = self.repo.reproduce(**self._common_kwargs, **self._repro_kwargs)
if len(stages) == 0:
ui.write(CmdDataStatus.UP_TO_DATE_MSG)
else:
@@ -55,16 +53,14 @@ def add_arguments(repro_parser):
repro_parser.add_argument(
"targets",
nargs="*",
help=(
"""\
help="""\
Stages to reproduce. 'dvc.yaml' by default.
The targets can be path to a dvc.yaml file or `.dvc` file,
or a stage name from dvc.yaml file from
current working directory. To run a stage from dvc.yaml
from other directories, the target must be a path followed by colon `:`
and then the stage name name.
"""
),
""",
).complete = completion.DVCFILES_AND_STAGE
repro_parser.add_argument(
"-f",
@@ -85,16 +81,14 @@ and then the stage name name.
"--single-item",
action="store_true",
default=False,
help="Reproduce only single data item without recursive dependencies "
"check.",
help="Reproduce only single data item without recursive dependencies check.",
)
repro_parser.add_argument(
"-p",
"--pipeline",
action="store_true",
default=False,
help="Reproduce the whole pipeline that the specified targets "
"belong to.",
help="Reproduce the whole pipeline that the specified targets belong to.",
)
repro_parser.add_argument(
"-P",
@@ -127,8 +121,10 @@ and then the stage name name.
"--force-downstream",
action="store_true",
default=False,
help="Reproduce all descendants of a changed stage even if their "
"direct dependencies didn't change.",
help=(
"Reproduce all descendants of a changed stage even if their "
"direct dependencies didn't change."
),
)
repro_parser.add_argument(
"--pull",
@@ -143,15 +139,14 @@ and then the stage name name.
"--dry",
action="store_true",
default=False,
help="Only print the commands that would be executed without "
"actually executing.",
help=(
"Only print the commands that would be executed without actually executing."
),
)
def add_parser(subparsers, parent_parser):
REPRO_HELP = (
"Reproduce complete or partial pipelines by executing their stages."
)
REPRO_HELP = "Reproduce complete or partial pipelines by executing their stages."
repro_parser = subparsers.add_parser(
"repro",
parents=[parent_parser],
+3 -7
View File
@@ -39,7 +39,7 @@ class CmdRun(CmdBase):
{
"cmd": parse_cmd(self.args.command),
"fname": kwargs.pop("file"),
"no_exec": (self.args.no_exec or bool(self.args.checkpoints)),
"no_exec": self.args.no_exec or bool(self.args.checkpoints),
"run_cache": not kwargs.pop("no_run_cache"),
}
)
@@ -55,9 +55,7 @@ class CmdRun(CmdBase):
def add_parser(subparsers, parent_parser):
from dvc.commands.stage import _add_common_args
RUN_HELP = (
"Generate a dvc.yaml file from a command and execute the command."
)
RUN_HELP = "Generate a dvc.yaml file from a command and execute the command."
run_parser = subparsers.add_parser(
"run",
parents=[parent_parser],
@@ -66,9 +64,7 @@ def add_parser(subparsers, parent_parser):
formatter_class=argparse.RawDescriptionHelpFormatter,
)
run_parser.add_argument("-n", "--name", help="Stage name.")
run_parser.add_argument(
"--file", metavar="<filename>", help=argparse.SUPPRESS
)
run_parser.add_argument("--file", metavar="<filename>", help=argparse.SUPPRESS)
run_parser.add_argument(
"--single-stage",
action="store_true",
+15 -17
View File
@@ -44,9 +44,7 @@ def generate_description(stage: "Stage") -> str:
return "; ".join(desc)
def prepare_description(
stage: "Stage", max_length: int = MAX_TEXT_LENGTH
) -> str:
def prepare_description(stage: "Stage", max_length: int = MAX_TEXT_LENGTH) -> str:
desc = stage.short_description() or generate_description(stage)
return truncate_text(desc, max_length)
@@ -57,9 +55,9 @@ def prepare_stages_data(
max_length: int = MAX_TEXT_LENGTH,
) -> Dict[str, str]:
return {
stage.addressing: prepare_description(stage, max_length=max_length)
if description
else ""
stage.addressing: (
prepare_description(stage, max_length=max_length) if description else ""
)
for stage in stages
}
@@ -75,9 +73,7 @@ class CmdStageList(CmdBase):
# removing duplicates while maintaining order
collected = chain.from_iterable(
self.repo.stage.collect(
target=target, recursive=self.args.recursive
)
self.repo.stage.collect(target=target, recursive=self.args.recursive)
for target in self.args.targets
)
return dict.fromkeys(collected).keys()
@@ -173,8 +169,7 @@ def _add_common_args(parser):
"--outs-no-cache",
action="append",
default=[],
help="Declare output file or directory "
"(do not put into DVC cache).",
help="Declare output file or directory (do not put into DVC cache).",
metavar="<filename>",
).complete = completion.FILE
parser.add_argument(
@@ -182,8 +177,10 @@ def _add_common_args(parser):
"--checkpoints",
action="append",
default=[],
help="Declare checkpoint output file or directory for 'dvc exp run'. "
"Not compatible with 'dvc repro'.",
help=(
"Declare checkpoint output file or directory for 'dvc exp run'. "
"Not compatible with 'dvc repro'."
),
metavar="<filename>",
).complete = completion.FILE
parser.add_argument(
@@ -196,16 +193,17 @@ def _add_common_args(parser):
"--outs-persist",
action="append",
default=[],
help="Declare output file or directory that will not be "
"removed upon repro.",
help="Declare output file or directory that will not be removed upon repro.",
metavar="<filename>",
)
parser.add_argument(
"--outs-persist-no-cache",
action="append",
default=[],
help="Declare output file or directory that will not be "
"removed upon repro (do not put into DVC cache).",
help=(
"Declare output file or directory that will not be "
"removed upon repro (do not put into DVC cache)."
),
metavar="<filename>",
)
parser.add_argument(
+1 -3
View File
@@ -81,9 +81,7 @@ class CmdDataStatus(CmdDataBase):
if not self.repo.index.stages:
ui.write(self.EMPTY_PROJECT_MSG)
elif self.args.cloud or self.args.remote:
remote = self.args.remote or self.repo.config["core"].get(
"remote"
)
remote = self.args.remote or self.repo.config["core"].get("remote")
ui.write(self.IN_SYNC_MSG.format(remote=remote))
else:
ui.write(self.UP_TO_DATE_MSG)
+4 -2
View File
@@ -61,8 +61,10 @@ def add_parser(subparsers, parent_parser):
"--no-download",
action="store_true",
default=False,
help="Update .dvc file git revision/hash value(s)"
" but do not download the file(s).",
help=(
"Update .dvc file git revision/hash value(s)"
" but do not download the file(s)."
),
)
update_parser.add_argument(
"--to-remote",
+1 -3
View File
@@ -21,9 +21,7 @@ class CmdVersion(CmdBaseNoRepo):
def add_parser(subparsers, parent_parser):
VERSION_HELP = (
"Display the DVC version and system/environment information."
)
VERSION_HELP = "Display the DVC version and system/environment information."
version_parser = subparsers.add_parser(
"version",
parents=[parent_parser],
+9 -26
View File
@@ -64,9 +64,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
def keys(self) -> List[str]:
return self._keys
def _iter_col_row(
self, row: Sequence["CellT"]
) -> Iterator[Tuple["CellT", Column]]:
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[Tuple["CellT", Column]]:
for val, col in zip_longest(row, self.columns):
if col is None:
break
@@ -103,9 +101,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
...
@overload
def __setitem__(
self, item: slice, value: Iterable[Sequence["CellT"]]
) -> None:
def __setitem__(self, item: slice, value: Iterable[Sequence["CellT"]]) -> None:
...
def __setitem__(self, item, value) -> None:
@@ -113,8 +109,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
if isinstance(item, slice):
n = len(self.columns)
normalized_rows = (
chain(val, repeat(self._fill_value, n - len(val)))
for val in value
chain(val, repeat(self._fill_value, n - len(val))) for val in value
)
# we need to transpose those rows into columnar format
# as we work in terms of column-based arrays
@@ -212,9 +207,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
self, cols: Optional[Iterable[str]] = None
) -> Iterable[Dict[str, "CellT"]]:
keys = self.keys() if cols is None else set(cols)
return [
{k: self._columns[k][i] for k in keys} for i in range(len(self))
]
return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
def dropna( # noqa: C901
self,
@@ -224,13 +217,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
):
if axis not in ["rows", "cols"]:
raise ValueError(
f"Invalid 'axis' value {axis}."
"Choose one of ['rows', 'cols']"
f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']"
)
if how not in ["any", "all"]:
raise ValueError(
f"Invalid 'how' value {how}. Choose one of ['any', 'all']"
)
raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']")
match_line: Set = set()
match_any = True
@@ -258,11 +248,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
if axis == "rows":
for name in self.keys(): # noqa: SIM118
self._columns[name] = Column(
[
x
for n, x in enumerate(self._columns[name])
if n not in to_drop
]
[x for n, x in enumerate(self._columns[name]) if n not in to_drop]
)
else:
self.drop(*to_drop)
@@ -275,8 +261,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
):
if axis not in ["rows", "cols"]:
raise ValueError(
f"Invalid 'axis' value {axis}."
"Choose one of ['rows', 'cols']"
f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']"
)
if axis == "cols":
@@ -362,9 +347,7 @@ def diff_table(
for item, change in sorted(diff_in_file.items()):
old_value = with_value(change.get("old"), fill_value)
new_value = with_value(change.get("new"), fill_value)
diff_value = with_value(
change.get("diff", on_empty_diff), fill_value
)
diff_value = with_value(change.get("diff", on_empty_diff), fill_value)
td.append(
[
fname,
+2 -5
View File
@@ -153,9 +153,7 @@ class Config(dict):
with open(config_file, "w+", encoding="utf-8"):
return Config(dvc_dir)
def load(
self, validate: bool = True, config: Optional["DictStrAny"] = None
):
def load(self, validate: bool = True, config: Optional["DictStrAny"] = None):
"""Loads config from all the config files.
Raises:
@@ -376,6 +374,5 @@ def merge(into, update):
def _lower_keys(data):
return {
k.lower(): _lower_keys(v) if isinstance(v, dict) else v
for k, v in data.items()
k.lower(): _lower_keys(v) if isinstance(v, dict) else v for k, v in data.items()
}
+3 -16
View File
@@ -2,16 +2,7 @@ import os
from urllib.parse import urlparse
from funcy import walk_values
from voluptuous import (
All,
Any,
Coerce,
Invalid,
Lower,
Optional,
Range,
Schema,
)
from voluptuous import All, Any, Coerce, Invalid, Lower, Optional, Range, Schema
Bool = All(
Lower,
@@ -34,9 +25,7 @@ def supported_cache_type(types):
unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
if unsupported:
raise Invalid(
"Unsupported cache type(s): {}".format(", ".join(unsupported))
)
raise Invalid("Unsupported cache type(s): {}".format(", ".join(unsupported)))
return types
@@ -258,9 +247,7 @@ SCHEMA = {
"machine": {
str: {
"cloud": All(Lower, Choices("aws", "azure")),
"region": All(
Lower, Choices("us-west", "us-east", "eu-west", "eu-north")
),
"region": All(Lower, Choices("us-west", "us-east", "eu-west", "eu-north")),
"image": str,
"spot": Bool,
"spot_price": Coerce(float),
+1 -3
View File
@@ -40,9 +40,7 @@ def _spawn_windows(cmd, env):
startupinfo = STARTUPINFO()
startupinfo.dwFlags |= STARTF_USESHOWWINDOW
_popen(
cmd, env=env, creationflags=creationflags, startupinfo=startupinfo
)
_popen(cmd, env=env, creationflags=creationflags, startupinfo=startupinfo)
def _spawn_posix(cmd, env):
+1 -3
View File
@@ -272,8 +272,6 @@ def draw(vertices, edges):
vertex.view.h,
)
canvas.text(
int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
)
canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
return canvas.draw()
+3 -1
View File
@@ -111,8 +111,10 @@ class DataCloud:
for hash_info in status.missing
)
logger.warning(
(
"Some of the cache files do not exist neither locally "
"nor on remote. Missing cache files:\n%s",
"nor on remote. Missing cache files:\n%s"
),
missing_desc,
)
+1 -3
View File
@@ -90,6 +90,4 @@ def _merge_params(s_list):
def loads_params(stage, s_list):
d = _merge_params(s_list)
return [
ParamsDependency(stage, path, params) for path, params in d.items()
]
return [ParamsDependency(stage, path, params) for path, params in d.items()]
+2 -6
View File
@@ -47,9 +47,7 @@ class Dependency(Output):
self.def_path = self.fs.path.version_path(
self.def_path, self.meta.version_id
)
self.fs_path = self.fs.path.version_path(
self.fs_path, self.meta.version_id
)
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
def download(self, to, jobs=None):
fs_download(self.fs, self.fs_path, to, jobs=jobs)
@@ -57,9 +55,7 @@ class Dependency(Output):
def save(self):
super().save()
if self.fs.version_aware:
self.fs_path = self.fs.path.version_path(
self.fs_path, self.meta.version_id
)
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
def dumpd(self, **kwargs):
if self.fs.version_aware:
+1 -3
View File
@@ -154,9 +154,7 @@ class ParamsDependency(Dependency):
try:
return load_path(self.fs_path, self.repo.fs)
except ParseError as exc:
raise BadParamFileError(
f"Unable to read parameters from '{self}'"
) from exc
raise BadParamFileError(f"Unable to read parameters from '{self}'") from exc
def get_hash(self):
info = self.read_params()
+7 -22
View File
@@ -35,9 +35,7 @@ class RepoDependency(Dependency):
}
}
def __init__(
self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs
):
def __init__(self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs):
self.def_repo = def_repo
self._objs: Dict[str, "HashFile"] = {}
self._meta: Dict[str, "Meta"] = {}
@@ -100,17 +98,13 @@ class RepoDependency(Dependency):
# immutable, hence its impossible for checksum to change.
return False
def get_used_objs(
self, **kwargs
) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
def get_used_objs(self, **kwargs) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
used, _, _ = self._get_used_and_obj(**kwargs)
return used
def _get_used_and_obj(
self, obj_only: bool = False, **kwargs
) -> Tuple[
Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"
]:
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"]:
from dvc.config import NoRemoteError
from dvc.exceptions import NoOutputOrStageError, PathMissingError
from dvc.utils import as_posix
@@ -163,9 +157,7 @@ class RepoDependency(Dependency):
used_obj_ids[object_store].update(oid for _, _, oid in obj)
return used_obj_ids, meta, obj
def _check_circular_import(
self, odb: "ObjectDB", obj_ids: Set["HashInfo"]
) -> None:
def _check_circular_import(self, odb: "ObjectDB", obj_ids: Set["HashInfo"]) -> None:
from dvc.exceptions import CircularImportError
from dvc.fs.dvc import DVCFileSystem
from dvc_data.hashfile.db.reference import ReferenceHashFileDB
@@ -187,10 +179,7 @@ class RepoDependency(Dependency):
for obj in iter_objs():
if not isinstance(obj.fs, DVCFileSystem):
continue
if (
obj.fs.repo_url in checked_urls
or obj.fs.repo.root_dir in checked_urls
):
if obj.fs.repo_url in checked_urls or obj.fs.repo.root_dir in checked_urls:
continue
self_url = self.repo.url or self.repo.root_dir
if (
@@ -221,9 +210,7 @@ class RepoDependency(Dependency):
)
return meta
def _make_repo(
self, locked: bool = True, **kwargs
) -> "ContextManager[Repo]":
def _make_repo(self, locked: bool = True, **kwargs) -> "ContextManager[Repo]":
from dvc.external_repo import external_repo
d = self.def_repo
@@ -232,6 +219,4 @@ class RepoDependency(Dependency):
def _get_rev(self, locked: bool = True):
d = self.def_repo
return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(
self.PARAM_REV
)
return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(self.PARAM_REV)
+8 -25
View File
@@ -63,9 +63,7 @@ def is_valid_filename(path):
def is_dvc_file(path):
return os.path.isfile(path) and (
is_valid_filename(path) or is_lock_file(path)
)
return os.path.isfile(path) and (is_valid_filename(path) or is_lock_file(path))
def is_lock_file(path):
@@ -77,9 +75,7 @@ def is_git_ignored(repo, path):
from dvc.scm import NoSCMError
try:
return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(
path
)
return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(path)
except NoSCMError:
return False
@@ -152,9 +148,7 @@ class FileMixin:
# 4. when the file is git ignored
if not self.exists():
dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path)
raise StageFileDoesNotExistError(
self.path, dvc_ignored=dvc_ignored
)
raise StageFileDoesNotExistError(self.path, dvc_ignored=dvc_ignored)
self._verify_filename()
if not self.repo.fs.isfile(self.path):
@@ -271,9 +265,7 @@ class ProjectFile(FileMixin):
@staticmethod
def _check_if_parametrized(stage, action: str = "dump") -> None:
if stage.raw_data.parametrized:
raise ParametrizedDumpError(
f"cannot {action} a parametrized {stage}"
)
raise ParametrizedDumpError(f"cannot {action} a parametrized {stage}")
def _dump_pipeline_file(self, stage):
self._check_if_parametrized(stage)
@@ -286,9 +278,7 @@ class ProjectFile(FileMixin):
data["stages"] = data.get("stages", {})
existing_entry = stage.name in data["stages"]
action = "Modifying" if existing_entry else "Adding"
logger.info(
"%s stage '%s' in '%s'", action, stage.name, self.relpath
)
logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
if existing_entry:
orig_stage_data = data["stages"][stage.name]
@@ -300,9 +290,7 @@ class ProjectFile(FileMixin):
@property
def stage(self):
raise DvcException(
"ProjectFile has multiple stages. Please specify it's name."
)
raise DvcException("ProjectFile has multiple stages. Please specify it's name.")
@cached_property
def contents(self) -> Dict[str, Any]:
@@ -366,10 +354,7 @@ class ProjectFile(FileMixin):
def get_lockfile_schema(d):
from dvc.schema import (
COMPILED_LOCKFILE_V1_SCHEMA,
COMPILED_LOCKFILE_V2_SCHEMA,
)
from dvc.schema import COMPILED_LOCKFILE_V1_SCHEMA, COMPILED_LOCKFILE_V2_SCHEMA
schema = {
LOCKFILE_VERSION.V1: COMPILED_LOCKFILE_V1_SCHEMA,
@@ -422,9 +407,7 @@ class Lockfile(FileMixin):
with modify_yaml(self.path, fs=self.repo.fs) as data:
version = LOCKFILE_VERSION.from_dict(data)
if version == LOCKFILE_VERSION.V1:
logger.info(
"Migrating lock file '%s' from v1 to v2", self.relpath
)
logger.info("Migrating lock file '%s' from v1 to v2", self.relpath)
migrate_lock_v1_to_v2(data, self.latest_version_info)
else:
if not data:
+9 -20
View File
@@ -150,8 +150,9 @@ class CyclicGraphError(DvcException):
class ConfirmRemoveError(DvcException):
def __init__(self, path):
super().__init__(
"unable to remove '{}' without a confirmation. Use "
"`-f` to force.".format(path)
"unable to remove '{}' without a confirmation. Use `-f` to force.".format(
path
)
)
@@ -177,9 +178,7 @@ class BadMetricError(DvcException):
class RecursiveAddingWhileUsingFilename(DvcException):
def __init__(self):
super().__init__(
"cannot use `fname` with multiple targets or `-R|--recursive`"
)
super().__init__("cannot use `fname` with multiple targets or `-R|--recursive`")
class OverlappingOutputPathsError(DvcException):
@@ -206,9 +205,7 @@ class FileMissingError(DvcException):
def __init__(self, path, hint=None):
self.path = path
hint = "" if hint is None else f". {hint}"
super().__init__(
f"Can't find '{path}' neither locally nor on remote{hint}"
)
super().__init__(f"Can't find '{path}' neither locally nor on remote{hint}")
class FileTransferError(DvcException):
@@ -250,9 +247,7 @@ class CollectCacheError(DvcException):
class NoRemoteInExternalRepoError(DvcException):
def __init__(self, url):
super().__init__(
f"No DVC remote is specified in target repository '{url}'."
)
super().__init__(f"No DVC remote is specified in target repository '{url}'.")
class NoOutputInExternalRepoError(DvcException):
@@ -277,8 +272,7 @@ class PathMissingError(DvcException):
" neither as a DVC output nor as a Git-tracked file."
)
default_msg_dvc_only = (
"The path '{}' does not exist in the target repository '{}'"
" as an DVC output."
"The path '{}' does not exist in the target repository '{}' as an DVC output."
)
def __init__(self, path, repo, dvc_only=False):
@@ -294,7 +288,6 @@ class URLMissingError(DvcException):
class RemoteCacheRequiredError(DvcException):
def __init__(self, scheme, fs_path):
super().__init__(
(
"Current operation was unsuccessful because '{}' requires "
@@ -319,8 +312,7 @@ class NoOutputOrStageError(DvcException):
def __init__(self, target, file):
super().__init__(
f"'{target}' "
f"does not exist as an output or a stage name in '{file}'"
f"'{target}' does not exist as an output or a stage name in '{file}'"
)
@@ -329,11 +321,8 @@ class MergeError(DvcException):
class CacheLinkError(DvcException):
SUPPORT_LINK = "See {} for more information.".format(
format_link(
"https://dvc.org/doc/user-guide/troubleshooting#cache-types"
)
format_link("https://dvc.org/doc/user-guide/troubleshooting#cache-types")
)
def __init__(self, fs_paths):
+3 -9
View File
@@ -88,9 +88,7 @@ def external_repo(
raise NoRemoteInExternalRepoError(url) from exc
except OutputNotFoundError as exc:
if exc.repo is repo:
raise NoOutputInExternalRepoError(
exc.output, repo.root_dir, url
) from exc
raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
raise
except FileMissingError as exc:
raise PathMissingError(exc.path, url) from exc
@@ -234,9 +232,7 @@ def _clone_default_branch(url, rev, for_write=False): # noqa: C901
os.path.join(clone_path, Git.GIT_DIR, "shallow")
)
if shallow:
logger.debug(
"erepo: using shallow clone for branch '%s'", rev
)
logger.debug("erepo: using shallow clone for branch '%s'", rev)
except CloneError:
git_dir = os.path.join(clone_path, ".git")
if os.path.exists(git_dir):
@@ -292,8 +288,6 @@ def _remove(path):
try:
os_retry(remove)(path)
except PermissionError:
logger.warning(
"Failed to remove '%s'", relpath(path), exc_info=True
)
logger.warning("Failed to remove '%s'", relpath(path), exc_info=True)
else:
remove(path)
+1 -3
View File
@@ -72,9 +72,7 @@ def get_fs_config(config, **kwargs):
except KeyError:
from dvc.config import RemoteNotFoundError
raise RemoteNotFoundError( # noqa: B904
f"remote '{name}' doesn't exist"
)
raise RemoteNotFoundError(f"remote '{name}' doesn't exist") # noqa: B904
else:
remote_conf = kwargs
return _resolve_remote_refs(config, remote_conf)
+1 -3
View File
@@ -58,9 +58,7 @@ class RichCallback(Callback):
@cached_property
def task(self) -> "TaskID":
return self.progress.add_task(
**self._task_kwargs # type: ignore[arg-type]
)
return self.progress.add_task(**self._task_kwargs) # type: ignore[arg-type]
def __enter__(self):
return self
+5 -22
View File
@@ -6,16 +6,7 @@ import os
import posixpath
import threading
from contextlib import suppress
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Optional,
Tuple,
Type,
Union,
)
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
from fsspec.spec import AbstractFileSystem
from funcy import wrap_with
@@ -126,9 +117,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
super().__init__()
if repo is None:
repo = self._make_repo(
url=url, rev=rev, subrepos=subrepos, **repo_kwargs
)
repo = self._make_repo(url=url, rev=rev, subrepos=subrepos, **repo_kwargs)
assert repo is not None
# pylint: disable=protected-access
repo_factory = repo._fs_conf["repo_factory"]
@@ -144,9 +133,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
relparts = ()
assert repo is not None
if repo.fs.path.isin(repo.fs.path.getcwd(), repo.root_dir):
relparts = repo.fs.path.relparts(
repo.fs.path.getcwd(), repo.root_dir
)
relparts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.root_dir)
return self.root_marker + self.sep.join(relparts)
self.path = Path(self.sep, getcwd=_getcwd)
@@ -222,9 +209,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
scm=self.repo.scm,
repo_factory=self.repo_factory,
)
self._datafss[key] = DataFileSystem(
index=repo.index.data["repo"]
)
self._datafss[key] = DataFileSystem(index=repo.index.data["repo"])
self._subrepos_trie[key] = repo
def _is_dvc_repo(self, dir_path):
@@ -331,9 +316,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
ignore_subrepos = kwargs.get("ignore_subrepos", True)
return self._info(key, path, ignore_subrepos=ignore_subrepos)
def _info( # noqa: C901
self, key, path, ignore_subrepos=True, check_ignored=True
):
def _info(self, key, path, ignore_subrepos=True, check_ignored=True): # noqa: C901
repo, dvc_fs, subkey = self._get_subrepo_info(key)
dvc_info = None
+10 -26
View File
@@ -30,9 +30,7 @@ class DvcIgnorePatterns(DvcIgnore):
from pathspec.patterns.gitwildmatch import _DIR_MARK
if pattern_list and isinstance(pattern_list[0], str):
pattern_list = [
PatternInfo(pattern, "") for pattern in pattern_list
]
pattern_list = [PatternInfo(pattern, "") for pattern in pattern_list]
self.sep = sep
self.pattern_list = pattern_list
@@ -40,20 +38,14 @@ class DvcIgnorePatterns(DvcIgnore):
self.regex_pattern_list = []
for count, pattern in enumerate(pattern_list):
pattern, group = GitWildMatchPattern.pattern_to_regex(
pattern.patterns
)
pattern, group = GitWildMatchPattern.pattern_to_regex(pattern.patterns)
if pattern:
pattern = pattern.replace(
f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>"
)
pattern = pattern.replace(f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>")
self.regex_pattern_list.append((pattern, group))
self.ignore_spec = [
(ignore, re.compile("|".join(item[0] for item in group)))
for ignore, group in groupby(
self.regex_pattern_list, lambda x: x[1]
)
for ignore, group in groupby(self.regex_pattern_list, lambda x: x[1])
if ignore is not None
]
@@ -64,9 +56,7 @@ class DvcIgnorePatterns(DvcIgnore):
with fs.open(path, encoding="utf-8") as fobj:
path_spec_lines = [
PatternInfo(line, f"{name}:{line_no + 1}:{line}")
for line_no, line in enumerate(
map(str.strip, fobj.readlines())
)
for line_no, line in enumerate(map(str.strip, fobj.readlines()))
if line and not (line.strip().startswith("#"))
]
@@ -158,9 +148,7 @@ class DvcIgnorePatterns(DvcIgnore):
return bool(self.pattern_list)
CheckIgnoreResult = namedtuple(
"CheckIgnoreResult", ["file", "match", "patterns"]
)
CheckIgnoreResult = namedtuple("CheckIgnoreResult", ["file", "match", "patterns"])
def _no_match(path):
@@ -249,9 +237,7 @@ class DvcIgnoreFilter:
dnames = []
for dname in dnames:
self._update_sub_repo(
self.fs.path.join(dirname, dname), ignore_trie
)
self._update_sub_repo(self.fs.path.join(dirname, dname), ignore_trie)
def _update_sub_repo(self, path, ignore_trie: Trie):
from dvc.repo import Repo
@@ -409,9 +395,7 @@ class DvcIgnoreFilter:
# https://github.com/iterative/dvc/issues/5046
full_target = self.fs.path.abspath(target)
if not self._outside_repo(full_target):
dirname, basename = self.fs.path.split(
self.fs.path.normpath(full_target)
)
dirname, basename = self.fs.path.split(self.fs.path.normpath(full_target))
pattern = self._get_trie_pattern(dirname)
if pattern:
matches = pattern.matches(
@@ -433,9 +417,9 @@ class DvcIgnoreFilter:
return self.is_ignored_file(path, ignore_subrepos)
if fs.isdir(path):
return self.is_ignored_dir(path, ignore_subrepos)
return self.is_ignored_file(
return self.is_ignored_file(path, ignore_subrepos) or self.is_ignored_dir(
path, ignore_subrepos
) or self.is_ignored_dir(path, ignore_subrepos)
)
def init(path):
+2 -4
View File
@@ -30,8 +30,7 @@ def get_dvc_info():
info = [
dvc_version,
"-" * len(dvc_version),
f"Platform: Python {platform.python_version()} on "
f"{platform.platform()}",
f"Platform: Python {platform.python_version()} on {platform.platform()}",
f"Subprojects:{_get_subprojects()}",
f"Supports:{_get_supported_remotes()}",
]
@@ -125,8 +124,7 @@ def _get_supported_remotes():
dependencies = []
for requirement in fs_cls.REQUIRES:
dependencies.append(
f"{requirement} = "
f"{importlib_metadata.version(requirement)}"
f"{requirement} = {importlib_metadata.version(requirement)}"
)
remote_info = scheme
+3 -9
View File
@@ -20,9 +20,7 @@ FAILED_TO_LOCK_MESSAGE = (
"Unable to acquire lock. Most likely another DVC process is running or "
"was terminated abruptly. Check the page {} for other possible reasons "
"and to learn how to resolve this."
).format(
format_link("https://dvc.org/doc/user-guide/troubleshooting#lock-issue")
)
).format(format_link("https://dvc.org/doc/user-guide/troubleshooting#lock-issue"))
class LockError(DvcException):
@@ -61,9 +59,7 @@ class LockBase(ABC):
class LockNoop(LockBase):
def __init__(
self, *args, **kwargs
): # pylint: disable=super-init-not-called
def __init__(self, *args, **kwargs): # pylint: disable=super-init-not-called
self._lock = False
def lock(self):
@@ -107,10 +103,8 @@ class Lock(LockBase):
with Tqdm(
bar_format="{desc}",
disable=not self._friendly,
desc=(
"If DVC froze, see `hardlink_lock` in {}".format(
desc="If DVC froze, see `hardlink_lock` in {}".format(
format_link("https://man.dvc.org/config#core")
)
),
):
self._lock = zc.lockfile.LockFile(self._lockfile)
+3 -12
View File
@@ -159,9 +159,7 @@ class LoggerHandler(logging.StreamHandler):
pass # noqa: S110
msg = self.format(record)
Tqdm.write(
msg, file=self.stream, end=getattr(self, "terminator", "\n")
)
Tqdm.write(msg, file=self.stream, end=getattr(self, "terminator", "\n"))
self.flush()
except (BrokenPipeError, RecursionError):
raise
@@ -171,9 +169,7 @@ class LoggerHandler(logging.StreamHandler):
def _is_verbose():
return (
logging.NOTSET
< logging.getLogger("dvc").getEffectiveLevel()
<= logging.DEBUG
logging.NOTSET < logging.getLogger("dvc").getEffectiveLevel() <= logging.DEBUG
)
@@ -186,17 +182,12 @@ def _iter_causes(exc):
def _stack_trace(exc_info):
import traceback
return (
"\n"
"{red}{line}{nc}\n"
"{trace}"
"{red}{line}{nc}".format(
return "\n{red}{line}{nc}\n{trace}{red}{line}{nc}".format(
red=colorama.Fore.RED,
line="-" * 60,
trace="".join(traceback.format_exception(*exc_info)),
nc=colorama.Fore.RESET,
)
)
def disable_other_loggers():
+1 -3
View File
@@ -67,9 +67,7 @@ class MachineBackends(Mapping):
initialized = self.initialized.get(key)
if not initialized:
backend = self.backends[key]
initialized = backend(
os.path.join(self.tmp_dir, key), **self.kwargs
)
initialized = backend(os.path.join(self.tmp_dir, key), **self.kwargs)
self.initialized[key] = initialized
return initialized
+1 -3
View File
@@ -21,9 +21,7 @@ class BaseMachineBackend(ABC):
"""Stop and destroy all instances of the specified machine."""
@abstractmethod
def instances(
self, name: Optional[str] = None, **config
) -> Iterator[dict]:
def instances(self, name: Optional[str] = None, **config) -> Iterator[dict]:
"""Iterate over status of all instances of the specified machine."""
def close(self): # noqa: B027
+2 -6
View File
@@ -56,12 +56,8 @@ class TerraformBackend(BaseMachineBackend):
create = partialmethod(_tpi_func, "create") # type: ignore[assignment]
destroy = partialmethod(_tpi_func, "destroy") # type: ignore[assignment]
instances = partialmethod(
_tpi_func, "instances"
) # type: ignore[assignment]
run_shell = partialmethod(
_tpi_func, "run_shell"
) # type: ignore[assignment]
instances = partialmethod(_tpi_func, "instances") # type: ignore[assignment]
run_shell = partialmethod(_tpi_func, "run_shell") # type: ignore[assignment]
def get_executor_kwargs(self, name: str, **config) -> dict:
with self.make_tpi(name) as tpi:
+13 -38
View File
@@ -151,9 +151,7 @@ def _merge_data(s_list):
d[key].update({})
continue
if not isinstance(key, dict):
raise ValueError( # noqa: TRY004
f"'{type(key).__name__}' not supported."
)
raise ValueError(f"'{type(key).__name__}' not supported.") # noqa: TRY004
for k, flags in key.items():
if not isinstance(flags, dict):
@@ -336,9 +334,7 @@ class Output:
if fs_config is not None:
fs_kwargs.update(**fs_config)
fs_cls, fs_config, fs_path = get_cloud_fs(
self.repo, url=path, **fs_kwargs
)
fs_cls, fs_config, fs_path = get_cloud_fs(self.repo, url=path, **fs_kwargs)
self.fs = fs_cls(**fs_config)
if (
@@ -500,9 +496,7 @@ class Output:
@property
def cache_path(self):
return self.odb.fs.unstrip_protocol(
self.odb.oid_to_path(self.hash_info.value)
)
return self.odb.fs.unstrip_protocol(self.odb.oid_to_path(self.hash_info.value))
def get_hash(self):
_, hash_info = self._get_hash_meta()
@@ -691,9 +685,7 @@ class Output:
dry_run=True,
)
if not self.IS_DEPENDENCY:
logger.debug(
"Output '%s' doesn't use cache. Skipping saving.", self
)
logger.debug("Output '%s' doesn't use cache. Skipping saving.", self)
self.hash_info = self.obj.hash_info
self.files = None
@@ -724,9 +716,7 @@ class Output:
if self.use_cache:
granular = (
self.is_dir_checksum
and filter_info
and filter_info != self.fs_path
self.is_dir_checksum and filter_info and filter_info != self.fs_path
)
if granular:
obj = self._commit_granular_dir(filter_info)
@@ -757,9 +747,7 @@ class Output:
self.set_exec()
def _commit_granular_dir(self, filter_info) -> Optional["HashFile"]:
prefix = self.fs.path.parts(
self.fs.path.relpath(filter_info, self.fs_path)
)
prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
staging, _, save_obj = build(
self.odb,
self.fs_path,
@@ -794,9 +782,7 @@ class Output:
ret.update(split_file_meta_from_cloud(meta_d))
if self.is_in_repo:
path = self.fs.path.as_posix(
relpath(self.fs_path, self.stage.wdir)
)
path = self.fs.path.as_posix(relpath(self.fs_path, self.stage.wdir))
else:
path = self.def_path
@@ -837,16 +823,13 @@ class Output:
if obj:
obj = cast("Tree", obj)
ret[self.PARAM_FILES] = [
split_file_meta_from_cloud(f)
for f in obj.as_list(with_meta=True)
split_file_meta_from_cloud(f) for f in obj.as_list(with_meta=True)
]
return ret
def verify_metric(self):
if self.fs.protocol != "local":
raise DvcException(
f"verify metric is not supported for {self.protocol}"
)
raise DvcException(f"verify metric is not supported for {self.protocol}")
if not self.metric:
return
@@ -902,9 +885,7 @@ class Output:
) -> Optional[Tuple[bool, Optional[bool]]]:
if not self.use_cache:
if progress_callback != DEFAULT_CALLBACK:
progress_callback.relative_update(
self.get_files_number(filter_info)
)
progress_callback.relative_update(self.get_files_number(filter_info))
return None
obj = self.get_obj(filter_info=filter_info)
@@ -1071,9 +1052,7 @@ class Output:
obj = self.get_obj()
if filter_info and filter_info != self.fs_path:
assert obj
prefix = self.fs.path.parts(
self.fs.path.relpath(filter_info, self.fs_path)
)
prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
obj = cast(Tree, obj)
return obj.filter(prefix)
return cast(Tree, obj)
@@ -1181,14 +1160,10 @@ class Output:
other.pop(opt, None)
if my != other:
raise MergeError(
"unable to auto-merge outputs with different options"
)
raise MergeError("unable to auto-merge outputs with different options")
if not out.is_dir_checksum:
raise MergeError(
"unable to auto-merge outputs that are not directories"
)
raise MergeError("unable to auto-merge outputs that are not directories")
def merge(self, ancestor, other, allowed=None):
from dvc_data.hashfile.tree import MergeError as TreeMergeError
+11 -18
View File
@@ -72,9 +72,7 @@ def _format_preamble(msg: str, path: str, spacing: str = " ") -> str:
def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn":
spacing = (
"\n"
if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded))
else " "
"\n" if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded)) else " "
)
message = _format_preamble(msg, path, spacing) + str(exc)
@@ -116,8 +114,7 @@ def check_interpolations(data: "DictStrAny", where: str, path: str):
def func(s: "DictStrAny") -> None:
if is_interpolated_string(s):
raise ResolveError(
_format_preamble(f"'{where}'", path)
+ "interpolating is not allowed"
_format_preamble(f"'{where}'", path) + "interpolating is not allowed"
)
return recurse(func)(data)
@@ -188,9 +185,7 @@ class DataResolver:
def resolve(self):
"""Used for testing purposes, otherwise use resolve_one()."""
data = join(map(self.resolve_one, self.get_keys()))
logger.trace( # type: ignore[attr-defined]
"Resolved dvc.yaml:\n%s", data
)
logger.trace("Resolved dvc.yaml:\n%s", data) # type: ignore[attr-defined]
return {STAGES_KWD: data}
def has_key(self, key: str):
@@ -203,9 +198,9 @@ class DataResolver:
return False
if key:
return isinstance(
definition, ForeachDefinition
) and definition.has_member(key)
return isinstance(definition, ForeachDefinition) and definition.has_member(
key
)
return not isinstance(definition, ForeachDefinition)
@collecting
@@ -309,9 +304,7 @@ class EntryDefinition:
value, skip_interpolation_checks=skip_checks, key=key
)
except (ParseError, KeyNotInContext) as exc:
format_and_raise(
exc, f"'{self.where}.{self.name}.{key}'", self.relpath
)
format_and_raise(exc, f"'{self.where}.{self.name}.{key}'", self.relpath)
class IterationPair(NamedTuple):
@@ -354,9 +347,7 @@ class ForeachDefinition:
try:
iterable = self.context.resolve(self.foreach_data, unwrap=False)
except (ContextError, ParseError) as exc:
format_and_raise(
exc, f"'{self.where}.{self.name}.foreach'", self.relpath
)
format_and_raise(exc, f"'{self.where}.{self.name}.foreach'", self.relpath)
# foreach data can be a resolved dictionary/list.
self._check_is_map_or_seq(iterable)
@@ -381,8 +372,10 @@ class ForeachDefinition:
if warn_for:
linking_verb = "is" if len(warn_for) == 1 else "are"
logger.warning(
(
"%s %s already specified, "
"will be overwritten for stages generated from '%s'",
"will be overwritten for stages generated from '%s'"
),
" and ".join(warn_for),
linking_verb,
self.name,
+7 -22
View File
@@ -48,9 +48,7 @@ class MergeError(ContextError):
self.key = key
to_node = into[key]
if not isinstance(to_node, Node) or not isinstance(new, Node):
super().__init__(
f"cannot merge '{key}' as it already exists in {into}"
)
super().__init__(f"cannot merge '{key}' as it already exists in {into}")
return
assert isinstance(to_node, Node)
@@ -138,9 +136,7 @@ class Node:
@dataclass
class Value(Node):
_value: Any
meta: Meta = field(
compare=False, default_factory=_default_meta, repr=False
)
meta: Meta = field(compare=False, default_factory=_default_meta, repr=False)
def __repr__(self):
return repr(self._value)
@@ -182,10 +178,7 @@ class Container(Node, ABC): # noqa: B024
assert meta
container = CtxDict if isinstance(value, dict) else CtxList
return container(value, meta=meta)
msg = (
"Unsupported value of type "
f"'{type(value).__name__}' in '{meta}'"
)
msg = f"Unsupported value of type '{type(value).__name__}' in '{meta}'"
raise TypeError(msg)
def __repr__(self):
@@ -219,9 +212,7 @@ class Container(Node, ABC): # noqa: B024
try:
d = self[index]
except LookupError as exc:
raise ValueError(
f"Could not find '{index}' in {self.data}"
) from exc
raise ValueError(f"Could not find '{index}' in {self.data}") from exc
if not rems:
return d
@@ -380,9 +371,7 @@ class Context(CtxDict):
data = {key: data[key] for key in select_keys}
except KeyError as exc:
key, *_ = exc.args
raise ParamsLoadError(
f"could not find '{key}' in '{path}'"
) from exc
raise ParamsLoadError(f"could not find '{key}' in '{path}'") from exc
meta = Meta(source=path, local=False)
ctx = cls(data, meta=meta)
@@ -480,9 +469,7 @@ class Context(CtxDict):
But for now, just `item` and `key`, this should do.
"""
# using dict to make the error messages ordered
new = dict.fromkeys(
[key for key in keys if key not in self._reserved_keys]
)
new = dict.fromkeys([key for key in keys if key not in self._reserved_keys])
self._reserved_keys.update(new)
try:
yield
@@ -542,9 +529,7 @@ class Context(CtxDict):
if is_exact_string(src, matches):
# replace "${enabled}", if `enabled` is a boolean, with it's actual
# value rather than it's string counterparts.
expr = get_expression(
matches[0], skip_checks=skip_interpolation_checks
)
expr = get_expression(matches[0], skip_checks=skip_interpolation_checks)
value = self.select(expr, unwrap=unwrap)
validate_value(value, key)
return value
+2 -7
View File
@@ -99,7 +99,6 @@ def _(obj: dict): # noqa: C901
result = ""
for k, v in flatten(obj).items():
if isinstance(v, bool):
if v:
result += f"--{k} "
@@ -115,9 +114,7 @@ def _(obj: dict): # noqa: C901
if isinstance(i, str):
i = escape_str(i)
elif isinstance(i, Iterable):
raise ParseError(
f"Cannot interpolate nested iterable in '{k}'"
)
raise ParseError(f"Cannot interpolate nested iterable in '{k}'")
if config.get("list", "nargs") == "append":
result += f"--{k} {i} "
@@ -206,9 +203,7 @@ def validate_value(value, key):
if not_primitive and not_foreach:
if isinstance(value, dict) and key == "cmd":
return True
raise ParseError(
f"Cannot interpolate data of type '{type(value).__name__}'"
)
raise ParseError(f"Cannot interpolate data of type '{type(value).__name__}'")
def str_interpolate(
+1 -3
View File
@@ -9,9 +9,7 @@ META_KWD = "meta"
def lockfile_version_schema(value):
expected = [LOCKFILE_VERSION.V2.value] # pylint: disable=no-member
msg = "invalid schema version {}, expected one of {}".format(
value, expected
)
msg = f"invalid schema version {value}, expected one of {expected}"
return validators.Any(*expected, msg=msg)(value)
+1 -3
View File
@@ -13,9 +13,7 @@ class ProcessNotFoundError(DvcException):
class TimeoutExpired(DvcException):
def __init__(self, cmd, timeout):
super().__init__(
f"'{cmd}' did not complete before timeout '{timeout}'"
)
super().__init__(f"'{cmd}' did not complete before timeout '{timeout}'")
self.cmd = cmd
self.timeout = timeout
+1 -3
View File
@@ -91,9 +91,7 @@ class ProcessManager:
raise UnsupportedSignalError(sig)
def handle_closed_process():
logging.warning(
"Process %s had already aborted unexpectedly.", name
)
logging.warning("Process %s had already aborted unexpectedly.", name)
process_info.returncode = -1
self[name] = process_info
+3 -11
View File
@@ -28,9 +28,7 @@ class Tqdm(tqdm):
"{postfix[info]}{n_fmt}/{total_fmt}"
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
)
BAR_FMT_NOTOTAL = (
"{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
)
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
BYTES_DEFAULTS = {
"unit": "B",
"unit_scale": True,
@@ -75,11 +73,7 @@ class Tqdm(tqdm):
if not disable:
disable = logger.getEffectiveLevel() > level
# auto-disable based on TTY
if (
not disable
and not env2bool(DVC_IGNORE_ISATTY)
and hasattr(file, "isatty")
):
if not disable and not env2bool(DVC_IGNORE_ISATTY) and hasattr(file, "isatty"):
disable = not file.isatty()
super().__init__(
iterable=iterable,
@@ -95,9 +89,7 @@ class Tqdm(tqdm):
if bar_format is None:
if self.__len__():
self.bar_format = (
self.BAR_FMT_DEFAULT_NESTED
if self.pos
else self.BAR_FMT_DEFAULT
self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT
)
else:
self.bar_format = self.BAR_FMT_NOTOTAL
+1 -3
View File
@@ -24,9 +24,7 @@ def ask(prompt: str, limited_to: Optional[Collection[str]] = None):
if answer in limited_to:
return answer
logger.info(
"Your response must be one of: %s. Please try again.", limited_to
)
logger.info("Your response must be one of: %s. Please try again.", limited_to)
def confirm(statement: str) -> bool:
+1 -7
View File
@@ -2,13 +2,7 @@ import json
from collections import defaultdict
from typing import Dict, List, Union
from dvc.render import (
REVISION_FIELD,
REVISIONS_KEY,
SRC_FIELD,
TYPE_KEY,
VERSION_FIELD,
)
from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD
from dvc.render.converter.image import ImageConverter
from dvc.render.converter.vega import VegaConverter
+5 -9
View File
@@ -81,9 +81,9 @@ def _is_datapoints(lst: List[Dict]):
to unexpected behavior
"""
return all(isinstance(item, dict) for item in lst) and set(
first(lst).keys()
) == {key for keys in lst for key in keys}
return all(isinstance(item, dict) for item in lst) and set(first(lst).keys()) == {
key for keys in lst for key in keys
}
def get_datapoints(file_content: Dict):
@@ -195,7 +195,6 @@ class VegaConverter(Converter):
return "x"
def flat_datapoints(self, revision): # noqa: C901
file2datapoints, properties = self.convert()
props_update = {}
@@ -218,7 +217,7 @@ class VegaConverter(Converter):
num_ys = len(ys)
if num_xs > 1 and num_xs != num_ys:
raise DvcException(
f"Cannot have different number of x and y data sources. Found "
"Cannot have different number of x and y data sources. Found "
f"{num_xs} x and {num_ys} y data sources."
)
@@ -316,9 +315,7 @@ def _update_from_field(
source_field = field
if len(source_datapoints) != len(target_datapoints):
raise IndexError(
"Source and target datapoints must have the same length"
)
raise IndexError("Source and target datapoints must have the same length")
for index, datapoint in enumerate(target_datapoints):
source_datapoint = source_datapoints[index]
@@ -332,6 +329,5 @@ def _update_from_index(datapoints: List[Dict], new_field: str):
def _update_all(datapoints: List[Dict], update_dict: Dict):
for datapoint in datapoints:
datapoint.update(update_dict)
+3 -10
View File
@@ -62,7 +62,6 @@ def match_defs_renderers(
out=None,
templates_dir: Optional["StrPath"] = None,
):
from dvc_render import ImageRenderer, VegaRenderer
plots_data = PlotsData(data)
@@ -80,9 +79,7 @@ def match_defs_renderers(
for rev, inner_id, plot_definition in group:
plot_sources = infer_data_sources(inner_id, plot_definition)
definitions_data = plots_data.get_definition_data(
plot_sources, rev
)
definitions_data = plots_data.get_definition_data(plot_sources, rev)
if ImageRenderer.matches(inner_id, None):
renderer_cls = ImageRenderer
@@ -91,9 +88,7 @@ def match_defs_renderers(
renderer_cls = VegaRenderer
renderer_id = plot_id
converter = _get_converter(
renderer_cls, inner_id, props, definitions_data
)
converter = _get_converter(renderer_cls, inner_id, props, definitions_data)
dps, rev_props = converter.flat_datapoints(rev)
if not final_props and rev_props:
@@ -103,7 +98,5 @@ def match_defs_renderers(
if "title" not in final_props:
final_props["title"] = renderer_id
if renderer_cls is not None:
renderers.append(
renderer_cls(plot_datapoints, renderer_id, **final_props)
)
renderers.append(renderer_cls(plot_datapoints, renderer_id, **final_props))
return renderers
+7 -27
View File
@@ -3,16 +3,7 @@ import os
from collections import defaultdict
from contextlib import contextmanager
from functools import wraps
from typing import (
TYPE_CHECKING,
Callable,
Iterable,
List,
Optional,
Tuple,
Union,
cast,
)
from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple, Union, cast
from dvc.exceptions import FileMissingError
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
@@ -148,9 +139,7 @@ class Repo:
import hashlib
root_dir_hash = hashlib.sha224(
self.root_dir.encode("utf-8")
).hexdigest()
root_dir_hash = hashlib.sha224(self.root_dir.encode("utf-8")).hexdigest()
db_dir = self.fs.path.join(
base_db_dir,
@@ -230,9 +219,7 @@ class Repo:
self.lock = make_lock(
self.fs.path.join(self.tmp_dir, "lock"),
tmp_dir=self.tmp_dir,
hardlink_lock=self.config["core"].get(
"hardlink_lock", False
),
hardlink_lock=self.config["core"].get("hardlink_lock", False),
friendly=True,
)
state_db_dir = self._get_database_dir("state")
@@ -338,8 +325,7 @@ class Repo:
from dvc.machine import MachineManager
if self.tmp_dir and (
self.config["feature"].get("machine", False)
or env2bool("DVC_TEST")
self.config["feature"].get("machine", False) or env2bool("DVC_TEST")
):
return MachineManager(self)
return None
@@ -393,14 +379,10 @@ class Repo:
return os.path.join(root_dir, cls.DVC_DIR)
@staticmethod
def init(
root_dir=os.curdir, no_scm=False, force=False, subdir=False
) -> "Repo":
def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo":
from dvc.repo.init import init
return init(
root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir
)
return init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir)
def unprotect(self, target):
return self.odb.repo.unprotect(target)
@@ -530,9 +512,7 @@ class Repo:
def dvcfs(self) -> "DVCFileSystem":
from dvc.fs.dvc import DVCFileSystem
return DVCFileSystem(
repo=self, subrepos=self.subrepos, **self._fs_conf
)
return DVCFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf)
@cached_property
def index_db_dir(self):
+2 -9
View File
@@ -15,12 +15,7 @@ from dvc.exceptions import (
)
from dvc.repo.scm_context import scm_context
from dvc.ui import ui
from dvc.utils import (
LARGE_DIR_SIZE,
glob_targets,
resolve_output,
resolve_paths,
)
from dvc.utils import LARGE_DIR_SIZE, glob_targets, resolve_output, resolve_paths
from dvc.utils.collections import ensure_list, validate
from . import locked
@@ -108,9 +103,7 @@ def translate_graph_error(stages: Stages) -> Iterator[None]:
def progress_iter(stages: Stages) -> Iterator["Stage"]:
total = len(stages)
desc = "Adding..."
with ui.progress(
stages, total=total, desc=desc, unit="file", leave=True
) as pbar:
with ui.progress(stages, total=total, desc=desc, unit="file", leave=True) as pbar:
if total == 1:
pbar.bar_format = desc
pbar.refresh()
+3 -9
View File
@@ -49,15 +49,11 @@ def brancher( # noqa: E302
repo_root_parts = ()
if self.fs.path.isin(self.root_dir, self.scm.root_dir):
repo_root_parts = self.fs.path.relparts(
self.root_dir, self.scm.root_dir
)
repo_root_parts = self.fs.path.relparts(self.root_dir, self.scm.root_dir)
cwd_parts = ()
if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir):
cwd_parts = self.fs.path.relparts(
self.fs.path.getcwd(), self.scm.root_dir
)
cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.scm.root_dir)
saved_fs = self.fs
saved_root = self.root_dir
@@ -91,9 +87,7 @@ def brancher( # noqa: E302
self.root_dir = self.fs.path.join("/", *repo_root_parts)
if cwd_parts:
cwd = self.fs.path.join( # type: ignore[unreachable]
"/", *cwd_parts
)
cwd = self.fs.path.join("/", *cwd_parts) # type: ignore[unreachable]
self.fs.path.chdir(cwd)
# ignore revs that don't contain repo root
+3 -12
View File
@@ -2,11 +2,7 @@ import logging
import os
from typing import TYPE_CHECKING, Dict, List, Set
from dvc.exceptions import (
CheckoutError,
CheckoutErrorSuggestGit,
NoOutputOrStageError,
)
from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit, NoOutputOrStageError
from dvc.utils import relpath
from . import locked
@@ -35,18 +31,13 @@ def _remove_unused_links(repo):
def get_all_files_numbers(pairs):
return sum(
stage.get_all_files_number(filter_info) for stage, filter_info in pairs
)
return sum(stage.get_all_files_number(filter_info) for stage, filter_info in pairs)
def _collect_pairs(
self: "Repo", targets, with_deps: bool, recursive: bool
) -> Set["StageInfo"]:
from dvc.stage.exceptions import (
StageFileBadNameError,
StageFileDoesNotExistError,
)
from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
pairs: Set["StageInfo"] = set()
for target in targets:
+2 -5
View File
@@ -30,8 +30,7 @@ def prompt_to_commit(stage, changes, force=False):
if not (force or prompt.confirm(_prepare_message(stage, changes))):
raise StageCommitError(
"unable to commit changed {}. Use `-f|--force` to "
"force.".format(stage)
f"unable to commit changed {stage}. Use `-f|--force` to force."
)
@@ -61,8 +60,6 @@ def commit(
if any(changes):
prompt_to_commit(stage, changes, force=force)
stage.save(allow_missing=allow_missing)
stage.commit(
filter_info=stage_info.filter_info, allow_missing=allow_missing
)
stage.commit(filter_info=stage_info.filter_info, allow_missing=allow_missing)
stage.dump(update_pipeline=False)
return [s.stage for s in stages_info]
+4 -11
View File
@@ -112,9 +112,7 @@ class GitInfo(TypedDict, total=False):
is_dirty: bool
def _git_info(
scm: Union["Git", "NoSCM"], untracked_files: str = "all"
) -> GitInfo:
def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitInfo:
from scmrepo.exceptions import SCMError
from dvc.scm import NoSCM
@@ -145,9 +143,7 @@ def _git_info(
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
from .index import build_data_index
workspace = build_data_index(
repo.index, repo.root_dir, repo.fs, compute_hash=True
)
workspace = build_data_index(repo.index, repo.root_dir, repo.fs, compute_hash=True)
return _diff(
repo.index.data["repo"],
@@ -180,9 +176,7 @@ class Status(TypedDict):
git: GitInfo
def _transform_git_paths_to_dvc(
repo: "Repo", files: Iterable[str]
) -> List[str]:
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
"""Transform files rel. to Git root to DVC root, and drop outside files."""
rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
@@ -237,8 +231,7 @@ def ls(
) -> Iterator[Dict[str, Any]]:
targets = targets or [None]
pairs = chain.from_iterable(
repo.stage.collect_granular(target, recursive=recursive)
for target in targets
repo.stage.collect_granular(target, recursive=recursive) for target in targets
)
for stage, filter_info in pairs:
for out in stage.filter_outs(filter_info):
+7 -19
View File
@@ -63,17 +63,13 @@ def diff(
targets_paths = None
if targets:
# convert targets to paths, and capture any missing targets
targets_paths, missing_targets[rev] = _targets_to_paths(
dvcfs, targets
)
targets_paths, missing_targets[rev] = _targets_to_paths(dvcfs, targets)
results[rev] = _paths_checksums(self, targets_paths)
if targets:
# check for overlapping missing targets between a_rev and b_rev
for target in set(missing_targets[a_rev]) & set(
missing_targets[b_rev]
):
for target in set(missing_targets[a_rev]) & set(missing_targets[b_rev]):
raise PathMissingError(target, self)
old = results[a_rev]
@@ -112,9 +108,7 @@ def diff(
if old[path] != new[path]
],
"renamed": renamed,
"not in cache": [
{"path": path, "hash": old[path]} for path in missing
],
"not in cache": [{"path": path, "hash": old[path]} for path in missing],
}
return ret if any(ret.values()) else {}
@@ -162,8 +156,7 @@ def _output_paths( # noqa: C901
continue
yield_output = targets is None or any(
output.fs.path.isin_or_eq(output.fs_path, target)
for target in targets
output.fs.path.isin_or_eq(output.fs_path, target) for target in targets
)
obj: Optional["HashFile"]
@@ -192,15 +185,12 @@ def _output_paths( # noqa: C901
or (
isinstance(targets, list)
and any(
output.fs.path.isin(target, output.fs_path)
for target in targets
output.fs.path.isin(target, output.fs_path) for target in targets
)
)
):
obj = cast("Tree", obj)
yield from _dir_output_paths(
output.fs, output.fs_path, obj, targets
)
yield from _dir_output_paths(output.fs, output.fs_path, obj, targets)
def _dir_output_paths(
@@ -277,8 +267,6 @@ def _calculate_renamed(
continue
old_path = old_paths.pop(index)
renamed.append(
{"path": {"old": old_path, "new": path}, "hash": path_hash}
)
renamed.append({"path": {"old": old_path, "new": path}, "hash": path_hash})
return renamed
+5 -14
View File
@@ -279,20 +279,15 @@ class Experiments:
branch: Optional[str] = None
try:
allow_multiple = bool(kwargs.get("params", None))
branch = self.get_branch_by_rev(
resume_rev, allow_multiple=allow_multiple
)
branch = self.get_branch_by_rev(resume_rev, allow_multiple=allow_multiple)
if not branch:
raise DvcException(
"Could not find checkpoint experiment "
f"'{resume_rev[:7]}'"
f"Could not find checkpoint experiment '{resume_rev[:7]}'"
)
baseline_rev = self._get_baseline(branch)
except MultipleBranchError as exc:
baselines = {
info.baseline_sha
for info in exc.ref_infos
if info.baseline_sha
info.baseline_sha for info in exc.ref_infos if info.baseline_sha
}
if len(baselines) == 1:
baseline_rev = baselines.pop()
@@ -341,9 +336,7 @@ class Experiments:
self.scm.remove_ref(EXEC_APPLY)
@unlocked_repo
def _reproduce_queue(
self, queue: "BaseStashQueue", **kwargs
) -> Dict[str, str]:
def _reproduce_queue(self, queue: "BaseStashQueue", **kwargs) -> Dict[str, str]:
"""Reproduce queued experiments.
Arguments:
@@ -416,9 +409,7 @@ class Experiments:
"""
result: Dict[str, Optional[str]] = {}
exclude = f"{EXEC_NAMESPACE}/*"
ref_dict = self.scm.describe(
revs, base=EXPS_NAMESPACE, exclude=exclude
)
ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude)
for rev in revs:
name: Optional[str] = None
ref = ref_dict[rev]
+5 -8
View File
@@ -8,11 +8,7 @@ from dvc.repo.scm_context import scm_context
from dvc.scm import Git
from dvc.utils.fs import remove
from .exceptions import (
ApplyConflictError,
BaselineMismatchError,
InvalidExpRevError,
)
from .exceptions import ApplyConflictError, BaselineMismatchError, InvalidExpRevError
from .executor.base import BaseExecutor
from .refs import EXEC_APPLY
@@ -43,7 +39,9 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901
(
exp_ref_info,
queue_entry,
) = exps.celery_queue.get_ref_and_entry_by_names(rev)[rev]
) = exps.celery_queue.get_ref_and_entry_by_names(
rev
)[rev]
baseline_sha = repo.scm.get_rev()
if exp_ref_info:
if baseline_sha != exp_ref_info.baseline_sha:
@@ -79,8 +77,7 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901
repo.scm.set_ref(EXEC_APPLY, exp_rev)
logger.info(
"Changes for experiment '%s' have been applied to your current "
"workspace.",
"Changes for experiment '%s' have been applied to your current workspace.",
rev,
)
+4 -4
View File
@@ -33,9 +33,11 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs):
break
if not ref_info:
msg = [
(
f"Ambiguous experiment name '{exp_rev}' can refer to "
"multiple experiments. To create a branch use a full "
"experiment ref:",
"experiment ref:"
),
"",
]
msg.extend([str(info) for info in ref_infos])
@@ -46,9 +48,7 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs):
branch_ref = f"refs/heads/{branch_name}"
if repo.scm.get_ref(branch_ref):
raise InvalidArgumentError(
f"Git branch '{branch_name}' already exists."
)
raise InvalidArgumentError(f"Git branch '{branch_name}' already exists.")
target = repo.scm.get_ref(str(ref_info))
repo.scm.set_ref(
+1 -3
View File
@@ -23,9 +23,7 @@ def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs):
rev = resolve_rev(repo.scm, b_rev)
new = collect_experiment_commit(repo, rev, param_deps=param_deps)
else:
new = collect_experiment_commit(
repo, "workspace", param_deps=param_deps
)
new = collect_experiment_commit(repo, "workspace", param_deps=param_deps)
with_unchanged = kwargs.pop("all", False)

Some files were not shown because too many files have changed in this diff Show More