update black and change line-length to 88 (#8953)

This commit is contained in:
skshetry
2023-02-04 05:28:07 +02:00
committed by GitHub
parent 5d2701341b
commit a86470ccbc
317 changed files with 1520 additions and 3738 deletions
+1 -1
View File
@@ -11,7 +11,7 @@ ignore=
P1 P1
# use `!r` instead of surrounding by quotes # use `!r` instead of surrounding by quotes
B028 B028
max_line_length = 79 max_line_length = 88
max-complexity = 10 max-complexity = 10
select = B,C,E,F,W,T4,B902,T,P,TYP select = B,C,E,F,W,T4,B902,T,P,TYP
show_source = true show_source = true
+1 -1
View File
@@ -35,7 +35,7 @@ repos:
args: args:
- --py38-plus - --py38-plus
- repo: https://github.com/psf/black - repo: https://github.com/psf/black
rev: 22.12.0 rev: 23.1.0
hooks: hooks:
- id: black - id: black
- repo: https://github.com/PyCQA/isort - repo: https://github.com/PyCQA/isort
+6 -18
View File
@@ -16,9 +16,7 @@ def viztracer_profile(
try: try:
import viztracer # pylint: disable=import-error import viztracer # pylint: disable=import-error
except ImportError: except ImportError:
print( # noqa: T201 print("Failed to run profiler, viztracer is not installed") # noqa: T201
"Failed to run profiler, viztracer is not installed"
)
yield yield
return return
@@ -80,9 +78,7 @@ def instrument(html_output=False):
try: try:
from pyinstrument import Profiler # pylint: disable=import-error from pyinstrument import Profiler # pylint: disable=import-error
except ImportError: except ImportError:
print( # noqa: T201 print("Failed to run profiler, pyinstrument is not installed") # noqa: T201
"Failed to run profiler, pyinstrument is not installed"
)
yield yield
return return
@@ -194,9 +190,7 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs):
path_func = _get_path_func("viztracer", "json") path_func = _get_path_func("viztracer", "json")
depth = kw.get("viztracer_depth") or -1 depth = kw.get("viztracer_depth") or -1
log_async = kw.get("viztracer_async") or False log_async = kw.get("viztracer_async") or False
prof = viztracer_profile( prof = viztracer_profile(path=path_func, depth=depth, log_async=log_async)
path=path_func, depth=depth, log_async=log_async
)
stack.enter_context(prof) stack.enter_context(prof)
yield yield
@@ -204,12 +198,8 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs):
def add_debugging_flags(parser): def add_debugging_flags(parser):
from argparse import SUPPRESS from argparse import SUPPRESS
parser.add_argument( parser.add_argument("--cprofile", action="store_true", default=False, help=SUPPRESS)
"--cprofile", action="store_true", default=False, help=SUPPRESS parser.add_argument("--yappi", action="store_true", default=False, help=SUPPRESS)
)
parser.add_argument(
"--yappi", action="store_true", default=False, help=SUPPRESS
)
parser.add_argument( parser.add_argument(
"--yappi-separate-threads", "--yappi-separate-threads",
action="store_true", action="store_true",
@@ -224,9 +214,7 @@ def add_debugging_flags(parser):
"--viztracer-async", action="store_true", default=False, help=SUPPRESS "--viztracer-async", action="store_true", default=False, help=SUPPRESS
) )
parser.add_argument("--cprofile-dump", help=SUPPRESS) parser.add_argument("--cprofile-dump", help=SUPPRESS)
parser.add_argument( parser.add_argument("--pdb", action="store_true", default=False, help=SUPPRESS)
"--pdb", action="store_true", default=False, help=SUPPRESS
)
parser.add_argument( parser.add_argument(
"--instrument", action="store_true", default=False, help=SUPPRESS "--instrument", action="store_true", default=False, help=SUPPRESS
) )
+2 -6
View File
@@ -34,9 +34,7 @@ def get_url(path, repo=None, rev=None, remote=None):
class _OpenContextManager(GCM): class _OpenContextManager(GCM):
def __init__( def __init__(self, func, args, kwds): # pylint: disable=super-init-not-called
self, func, args, kwds
): # pylint: disable=super-init-not-called
self.gen = func(*args, **kwds) self.gen = func(*args, **kwds)
self.func, self.args, self.kwds = ( # type: ignore[assignment] self.func, self.args, self.kwds = ( # type: ignore[assignment]
func, func,
@@ -45,9 +43,7 @@ class _OpenContextManager(GCM):
) )
def __getattr__(self, name): def __getattr__(self, name):
raise AttributeError( raise AttributeError("dvc.api.open() should be used in a with statement.")
"dvc.api.open() should be used in a with statement."
)
def open( # noqa, pylint: disable=redefined-builtin def open( # noqa, pylint: disable=redefined-builtin
+12 -13
View File
@@ -56,8 +56,10 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
if exc.errno == errno.EMFILE: if exc.errno == errno.EMFILE:
logger.exception( logger.exception(
"too many open files, please visit " (
"%s to see how to handle this problem", "too many open files, please visit "
"%s to see how to handle this problem"
),
error_link("many-files"), error_link("many-files"),
extra={"tb_only": True}, extra={"tb_only": True},
) )
@@ -80,7 +82,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
if cmd: if cmd:
link = format_link("https://dvc.org/doc/install") link = format_link("https://dvc.org/doc/install")
hint = ( hint = (
f"To install dvc with those dependencies, run:\n" "To install dvc with those dependencies, run:\n"
"\n" "\n"
f"\t{cmd}\n" f"\t{cmd}\n"
"\n" "\n"
@@ -91,8 +93,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
hint = f"\nPlease report this bug to {link}. Thank you!" hint = f"\nPlease report this bug to {link}. Thank you!"
logger.exception( logger.exception(
"URL '%s' is supported but requires these missing " "URL '%s' is supported but requires these missing dependencies: %s. %s",
"dependencies: %s. %s",
exc.url, exc.url,
exc.missing_deps, exc.missing_deps,
hint, hint,
@@ -118,9 +119,11 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
directory = relpath(exc.directory) directory = relpath(exc.directory)
logger.exception( logger.exception(
"Could not open pickled '%s' cache.\n" (
"Remove the '%s' directory and then retry this command." "Could not open pickled '%s' cache.\n"
"\nSee %s for more information.", "Remove the '%s' directory and then retry this command."
"\nSee %s for more information."
),
exc.type, exc.type,
directory, directory,
error_link("pickle"), error_link("pickle"),
@@ -181,11 +184,7 @@ def main(argv=None): # noqa: C901
set_loggers_level(level) set_loggers_level(level)
if level and level <= logging.DEBUG: if level and level <= logging.DEBUG:
from platform import ( from platform import platform, python_implementation, python_version
platform,
python_implementation,
python_version,
)
from dvc import __version__ from dvc import __version__
from dvc.utils.pkg import PKG from dvc.utils.pkg import PKG
+2 -6
View File
@@ -2,9 +2,7 @@ from argparse import _AppendAction
class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
def __call__( def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002
self, parser, namespace, values, option_string=None # noqa: ARG002
):
from funcy import ldistinct from funcy import ldistinct
items = getattr(namespace, self.dest) or [] items = getattr(namespace, self.dest) or []
@@ -13,9 +11,7 @@ class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
class KeyValueArgs(_AppendAction): class KeyValueArgs(_AppendAction):
def __call__( def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002
self, parser, namespace, values, option_string=None # noqa: ARG002
):
items = getattr(namespace, self.dest) or {} items = getattr(namespace, self.dest) or {}
for value in filter(bool, values): for value in filter(bool, values):
key, _, value = value.partition("=") key, _, value = value.partition("=")
+1 -3
View File
@@ -28,7 +28,5 @@ def hide_subparsers_from_help(subparsers):
# Need to set `add_help=False`, but avoid setting `help` # Need to set `add_help=False`, but avoid setting `help`
# (not even to `argparse.SUPPPRESS`). # (not even to `argparse.SUPPPRESS`).
# NOTE: The argument is the parent subparser, not the subcommand parser. # NOTE: The argument is the parent subparser, not the subcommand parser.
cmds = [ cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help]
cmd for cmd, parser in subparsers.choices.items() if parser.add_help
]
subparsers.metavar = "{{{}}}".format(",".join(cmds)) subparsers.metavar = "{{{}}}".format(",".join(cmds))
+1 -4
View File
@@ -41,10 +41,7 @@ def _add_annotating_args(parser: argparse.ArgumentParser) -> None:
class CmdAdd(CmdBase): class CmdAdd(CmdBase):
def run(self): def run(self):
from dvc.exceptions import ( from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
DvcException,
RecursiveAddingWhileUsingFilename,
)
try: try:
if len(self.args.targets) > 1 and self.args.file: if len(self.args.targets) > 1 and self.args.file:
+6 -4
View File
@@ -68,10 +68,12 @@ def add_parser(subparsers, parent_parser):
) )
cache_dir_parser.add_argument( cache_dir_parser.add_argument(
"value", "value",
help="Path to cache directory. Relative paths are resolved relative " help=(
"to the current directory and saved to config relative to the " "Path to cache directory. Relative paths are resolved relative "
"config file location. If no path is provided, it returns the " "to the current directory and saved to config relative to the "
"current cache directory.", "config file location. If no path is provided, it returns the "
"current cache directory."
),
nargs="?", nargs="?",
).complete = completion.DIR ).complete = completion.DIR
cache_dir_parser.set_defaults(func=CmdCacheDir) cache_dir_parser.set_defaults(func=CmdCacheDir)
+10 -10
View File
@@ -62,9 +62,7 @@ class CmdCheckIgnore(CmdBase):
raise DvcException("cannot have both `targets` and `--stdin`") raise DvcException("cannot have both `targets` and `--stdin`")
if self.args.non_matching and not self.args.details: if self.args.non_matching and not self.args.details:
raise DvcException( raise DvcException("`--non-matching` is only valid with `--details`")
"`--non-matching` is only valid with `--details`"
)
if self.args.all and not self.args.details: if self.args.all and not self.args.details:
raise DvcException("`--all` is only valid with `--details`") raise DvcException("`--all` is only valid with `--details`")
@@ -80,9 +78,7 @@ class CmdCheckIgnore(CmdBase):
def add_parser(subparsers, parent_parser): def add_parser(subparsers, parent_parser):
ADD_HELP = ( ADD_HELP = "Check whether files or directories are excluded due to `.dvcignore`."
"Check whether files or directories are excluded due to `.dvcignore`."
)
parser = subparsers.add_parser( parser = subparsers.add_parser(
"check-ignore", "check-ignore",
@@ -103,16 +99,20 @@ def add_parser(subparsers, parent_parser):
"--all", "--all",
action="store_true", action="store_true",
default=False, default=False,
help="Include the target paths which don't match any pattern " help=(
"in the `--details` list.", "Include the target paths which don't match any pattern "
"in the `--details` list."
),
) )
parser.add_argument( parser.add_argument(
"-n", "-n",
"--non-matching", "--non-matching",
action="store_true", action="store_true",
default=False, default=False,
help="Include the target paths which don't match any pattern " help=(
"in the `--details` list.", "Include the target paths which don't match any pattern "
"in the `--details` list."
),
) )
parser.add_argument( parser.add_argument(
"--stdin", "--stdin",
+2 -6
View File
@@ -22,9 +22,7 @@ def log_changes(stats):
continue continue
for entry in entries: for entry in entries:
ui.write( ui.write(f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t")
f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t"
)
class CmdCheckout(CmdBase): class CmdCheckout(CmdBase):
@@ -46,9 +44,7 @@ class CmdCheckout(CmdBase):
if self.args.summary: if self.args.summary:
default_message = "No changes." default_message = "No changes."
msg = get_summary( msg = get_summary(sorted(stats.items(), key=operator.itemgetter(0)))
sorted(stats.items(), key=operator.itemgetter(0))
)
ui.write(msg or default_message) ui.write(msg or default_message)
else: else:
log_changes(stats) log_changes(stats)
+1 -3
View File
@@ -24,9 +24,7 @@ class CmdCommit(CmdBase):
force=self.args.force, force=self.args.force,
) )
except DvcException: except DvcException:
logger.exception( logger.exception("failed to commit%s", (" " + target) if target else "")
"failed to commit%s", (" " + target) if target else ""
)
return 1 return 1
return 0 return 0
+3 -9
View File
@@ -17,9 +17,7 @@ def _name_type(value):
match = re.match(NAME_REGEX, value) match = re.match(NAME_REGEX, value)
if not match: if not match:
raise argparse.ArgumentTypeError( raise argparse.ArgumentTypeError(
"name argument should look like " "name argument should look like remote.name.option or section.option"
"remote.name.option or "
"section.option"
) )
return ( return (
bool(match.group("remote")), bool(match.group("remote")),
@@ -70,9 +68,7 @@ class CmdConfig(CmdBaseNoRepo):
for level in levels: for level in levels:
conf = self.config.read(level) conf = self.config.read(level)
prefix = self._config_file_prefix( prefix = self._config_file_prefix(self.args.show_origin, self.config, level)
self.args.show_origin, self.config, level
)
configs = list(self._format_config(conf, prefix)) configs = list(self._format_config(conf, prefix))
if configs: if configs:
ui.write("\n".join(configs)) ui.write("\n".join(configs))
@@ -131,9 +127,7 @@ class CmdConfig(CmdBaseNoRepo):
raise ConfigError(f"{name} '{section}' doesn't exist") raise ConfigError(f"{name} '{section}' doesn't exist")
if opt and opt not in conf[section]: if opt and opt not in conf[section]:
raise ConfigError( raise ConfigError(f"option '{opt}' doesn't exist in {name} '{section}'")
f"option '{opt}' doesn't exist in {name} '{section}'"
)
def _get_appropriate_levels(self, levels): def _get_appropriate_levels(self, levels):
if levels: if levels:
+5 -5
View File
@@ -86,9 +86,7 @@ def _collect_targets(repo, target, outs):
targets.extend([str(out) for out in stage.outs]) targets.extend([str(out) for out in stage.outs])
continue continue
for out in outs_trie.itervalues( # noqa: B301 for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)): # noqa: B301
prefix=repo.fs.path.parts(path)
):
targets.extend(str(out)) targets.extend(str(out))
return targets return targets
@@ -204,7 +202,9 @@ def add_parser(subparsers, parent_parser):
dag_parser.add_argument( dag_parser.add_argument(
"target", "target",
nargs="?", nargs="?",
help="Stage name or output to show pipeline for. " help=(
"Finds all stages in the workspace by default.", "Stage name or output to show pipeline for. "
"Finds all stages in the workspace by default."
),
) )
dag_parser.set_defaults(func=CmdDAG) dag_parser.set_defaults(func=CmdDAG)
+10 -19
View File
@@ -8,11 +8,7 @@ from funcy import chunks, compact, log_durations
from dvc.cli import completion from dvc.cli import completion
from dvc.cli.actions import CommaSeparatedArgs from dvc.cli.actions import CommaSeparatedArgs
from dvc.cli.command import CmdBase from dvc.cli.command import CmdBase
from dvc.cli.utils import ( from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help
append_doc_link,
fix_subparsers,
hide_subparsers_from_help,
)
from dvc.ui import ui from dvc.ui import ui
from dvc.utils import colorize from dvc.utils import colorize
@@ -39,20 +35,19 @@ class CmdDataStatus(CmdBase):
} }
HINTS = { HINTS = {
"not_in_cache": ('use "dvc fetch <file>..." to download files',), "not_in_cache": ('use "dvc fetch <file>..." to download files',),
"committed": ( "committed": ("git commit the corresponding dvc files to update the repo",),
"git commit the corresponding dvc files to update the repo",
),
"uncommitted": ( "uncommitted": (
'use "dvc commit <file>..." to track changes', 'use "dvc commit <file>..." to track changes',
'use "dvc checkout <file>..." to discard changes', 'use "dvc checkout <file>..." to discard changes',
), ),
"untracked": ( "untracked": (
'use "git add <file> ..." or ' (
'"dvc add <file>..." to commit to git or to dvc', 'use "git add <file> ..." or '
'"dvc add <file>..." to commit to git or to dvc'
),
), ),
"git_dirty": ( "git_dirty": (
"there are {}changes not tracked by dvc, " 'there are {}changes not tracked by dvc, use "git status" to see',
'use "git status" to see',
), ),
} }
@@ -96,8 +91,7 @@ class CmdDataStatus(CmdBase):
if isinstance(stage_status, dict): if isinstance(stage_status, dict):
items = [ items = [
": ".join([state, file]) ": ".join([state, file]) for file, state in stage_status.items()
for file, state in stage_status.items()
] ]
else: else:
items = stage_status items = stage_status
@@ -180,9 +174,7 @@ class CmdDataLs(CmdBase):
filter_labels = set(self.args.labels) filter_labels = set(self.args.labels)
filter_types = set(self.args.type) filter_types = set(self.args.type)
d = ls( d = ls(self.repo, targets=self.args.targets, recursive=self.args.recursive)
self.repo, targets=self.args.targets, recursive=self.args.recursive
)
self._show_table( self._show_table(
d, d,
filter_labels=filter_labels, filter_labels=filter_labels,
@@ -205,8 +197,7 @@ def add_parser(subparsers, parent_parser):
fix_subparsers(data_subparsers) fix_subparsers(data_subparsers)
DATA_STATUS_HELP = ( DATA_STATUS_HELP = (
"Show changes between the last git commit, " "Show changes between the last git commit, the dvcfiles and the workspace."
"the dvcfiles and the workspace."
) )
data_status_parser = data_subparsers.add_parser( data_status_parser = data_subparsers.add_parser(
"status", "status",
+13 -12
View File
@@ -262,9 +262,7 @@ def add_parser(subparsers, _parent_parser):
push_parser.set_defaults(func=CmdDataPush) push_parser.set_defaults(func=CmdDataPush)
# Fetch # Fetch
FETCH_HELP = ( FETCH_HELP = "Download files or directories from remote storage to the cache."
"Download files or directories from remote storage to the cache."
)
fetch_parser = subparsers.add_parser( fetch_parser = subparsers.add_parser(
"fetch", "fetch",
@@ -320,9 +318,7 @@ def add_parser(subparsers, _parent_parser):
fetch_parser.set_defaults(func=CmdDataFetch) fetch_parser.set_defaults(func=CmdDataFetch)
# Status # Status
STATUS_HELP = ( STATUS_HELP = "Show changed stages, compare local cache and a remote storage."
"Show changed stages, compare local cache and a remote storage."
)
status_parser = subparsers.add_parser( status_parser = subparsers.add_parser(
"status", "status",
@@ -360,24 +356,29 @@ def add_parser(subparsers, _parent_parser):
"--all-branches", "--all-branches",
action="store_true", action="store_true",
default=False, default=False,
help="Show status of a local cache compared to a remote repository " help=(
"for all branches.", "Show status of a local cache compared to a remote repository "
"for all branches."
),
) )
status_parser.add_argument( status_parser.add_argument(
"-T", "-T",
"--all-tags", "--all-tags",
action="store_true", action="store_true",
default=False, default=False,
help="Show status of a local cache compared to a remote repository " help=(
"for all tags.", "Show status of a local cache compared to a remote repository for all tags."
),
) )
status_parser.add_argument( status_parser.add_argument(
"-A", "-A",
"--all-commits", "--all-commits",
action="store_true", action="store_true",
default=False, default=False,
help="Show status of a local cache compared to a remote repository " help=(
"for all commits.", "Show status of a local cache compared to a remote repository "
"for all commits."
),
) )
status_parser.add_argument( status_parser.add_argument(
"-d", "-d",
+9 -14
View File
@@ -112,9 +112,7 @@ class CmdDiff(CmdBase):
return return
states_summary = ", ".join( states_summary = ", ".join(
f"{summary[state]} {state}" f"{summary[state]} {state}" for state in states if summary[state] > 0
for state in states
if summary[state] > 0
) )
ui.write("files summary:", states_summary) ui.write("files summary:", states_summary)
@@ -122,9 +120,7 @@ class CmdDiff(CmdBase):
from dvc.exceptions import DvcException from dvc.exceptions import DvcException
try: try:
diff = self.repo.diff( diff = self.repo.diff(self.args.a_rev, self.args.b_rev, self.args.targets)
self.args.a_rev, self.args.b_rev, self.args.targets
)
show_hash = self.args.show_hash show_hash = self.args.show_hash
hide_missing = self.args.b_rev or self.args.hide_missing hide_missing = self.args.b_rev or self.args.hide_missing
if hide_missing: if hide_missing:
@@ -133,9 +129,11 @@ class CmdDiff(CmdBase):
for key, entries in diff.items(): for key, entries in diff.items():
entries = sorted( entries = sorted(
entries, entries,
key=lambda entry: entry["path"]["old"] key=lambda entry: (
if isinstance(entry["path"], dict) entry["path"]["old"]
else entry["path"], if isinstance(entry["path"], dict)
else entry["path"]
),
) )
if not show_hash: if not show_hash:
for entry in entries: for entry in entries:
@@ -170,10 +168,7 @@ def add_parser(subparsers, parent_parser):
diff_parser.add_argument( diff_parser.add_argument(
"--targets", "--targets",
nargs="*", nargs="*",
help=( help="Specific DVC-tracked files to compare. Accepts one or more file paths.",
"Specific DVC-tracked files to compare. "
"Accepts one or more file paths."
),
metavar="<paths>", metavar="<paths>",
).complete = completion.FILE ).complete = completion.FILE
diff_parser.add_argument( diff_parser.add_argument(
@@ -184,7 +179,7 @@ def add_parser(subparsers, parent_parser):
) )
diff_parser.add_argument( diff_parser.add_argument(
"b_rev", "b_rev",
help=("New Git commit to compare (defaults to the current workspace)"), help="New Git commit to compare (defaults to the current workspace)",
nargs="?", nargs="?",
) )
diff_parser.add_argument( diff_parser.add_argument(
+2 -7
View File
@@ -1,10 +1,6 @@
import argparse import argparse
from dvc.cli.utils import ( from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help
append_doc_link,
fix_subparsers,
hide_subparsers_from_help,
)
from dvc.commands.experiments import ( from dvc.commands.experiments import (
apply, apply,
branch, branch,
@@ -54,8 +50,7 @@ def add_parser(subparsers, parent_parser):
experiments_subparsers = experiments_parser.add_subparsers( experiments_subparsers = experiments_parser.add_subparsers(
dest="cmd", dest="cmd",
help="Use `dvc experiments CMD --help` to display " help="Use `dvc experiments CMD --help` to display command-specific help.",
"command-specific help.",
) )
fix_subparsers(experiments_subparsers) fix_subparsers(experiments_subparsers)
+2 -8
View File
@@ -10,19 +10,13 @@ logger = logging.getLogger(__name__)
class CmdExperimentsApply(CmdBase): class CmdExperimentsApply(CmdBase):
def run(self): def run(self):
self.repo.experiments.apply(self.args.experiment, force=self.args.force)
self.repo.experiments.apply(
self.args.experiment, force=self.args.force
)
return 0 return 0
def add_parser(experiments_subparsers, parent_parser): def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_APPLY_HELP = "Apply the changes from an experiment to your workspace."
EXPERIMENTS_APPLY_HELP = (
"Apply the changes from an experiment to your workspace."
)
experiments_apply_parser = experiments_subparsers.add_parser( experiments_apply_parser = experiments_subparsers.add_parser(
"apply", "apply",
parents=[parent_parser], parents=[parent_parser],
+1 -5
View File
@@ -9,14 +9,12 @@ logger = logging.getLogger(__name__)
class CmdExperimentsBranch(CmdBase): class CmdExperimentsBranch(CmdBase):
def run(self): def run(self):
self.repo.experiments.branch(self.args.experiment, self.args.branch) self.repo.experiments.branch(self.args.experiment, self.args.branch)
return 0 return 0
def add_parser(experiments_subparsers, parent_parser): def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch." EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch."
experiments_branch_parser = experiments_subparsers.add_parser( experiments_branch_parser = experiments_subparsers.add_parser(
"branch", "branch",
@@ -28,7 +26,5 @@ def add_parser(experiments_subparsers, parent_parser):
experiments_branch_parser.add_argument( experiments_branch_parser.add_argument(
"experiment", help="Experiment to be promoted." "experiment", help="Experiment to be promoted."
) )
experiments_branch_parser.add_argument( experiments_branch_parser.add_argument("branch", help="Name for the new Git branch")
"branch", help="Name for the new Git branch"
)
experiments_branch_parser.set_defaults(func=CmdExperimentsBranch) experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)
+1 -5
View File
@@ -13,7 +13,6 @@ logger = logging.getLogger(__name__)
class CmdExperimentsDiff(CmdBase): class CmdExperimentsDiff(CmdBase):
def run(self): def run(self):
try: try:
diff = self.repo.experiments.diff( diff = self.repo.experiments.diff(
a_rev=self.args.a_rev, a_rev=self.args.a_rev,
@@ -53,10 +52,7 @@ class CmdExperimentsDiff(CmdBase):
def add_parser(experiments_subparsers, parent_parser): def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_DIFF_HELP = "Show changes between experiments in the DVC repository."
EXPERIMENTS_DIFF_HELP = (
"Show changes between experiments in the DVC repository."
)
experiments_diff_parser = experiments_subparsers.add_parser( experiments_diff_parser = experiments_subparsers.add_parser(
"diff", "diff",
+1 -4
View File
@@ -9,10 +9,7 @@ class CmdExecutorRun(CmdBaseNoRepo):
"""Run an experiment executor.""" """Run an experiment executor."""
def run(self): def run(self):
from dvc.repo.experiments.executor.base import ( from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo
BaseExecutor,
ExecutorInfo,
)
from dvc.utils.serialize import load_json from dvc.utils.serialize import load_json
info = ExecutorInfo.from_dict(load_json(self.args.infofile)) info = ExecutorInfo.from_dict(load_json(self.args.infofile))
+2 -7
View File
@@ -41,10 +41,7 @@ class CmdExperimentsGC(CmdBase):
msg += " and all git tags" msg += " and all git tags"
if self.args.commit_date: if self.args.commit_date:
msg += ( msg += f" and all git commits before date {self.args.commit_date}"
" and all git commits before date "
f"{self.args.commit_date}"
)
msg += " of the current repo." msg += " of the current repo."
if self.args.queued: if self.args.queued:
@@ -79,11 +76,9 @@ class CmdExperimentsGC(CmdBase):
def add_parser(experiments_subparsers, parent_parser): def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments." EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
EXPERIMENTS_GC_DESCRIPTION = ( EXPERIMENTS_GC_DESCRIPTION = (
"Removes all experiments which are not derived from the specified" "Removes all experiments which are not derived from the specifiedGit revisions."
"Git revisions."
) )
experiments_gc_parser = experiments_subparsers.add_parser( experiments_gc_parser = experiments_subparsers.add_parser(
"gc", "gc",
+27 -18
View File
@@ -92,9 +92,7 @@ class CmdExperimentsInit(CmdBase):
if new_out_dirs: if new_out_dirs:
out_dirs_paths = humanize.join(map(path_fmt, new_out_dirs)) out_dirs_paths = humanize.join(map(path_fmt, new_out_dirs))
ui.write( ui.write(f"Creating output directories: {out_dirs_paths}", styled=True)
f"Creating output directories: {out_dirs_paths}", styled=True
)
ui.write( ui.write(
f"Creating [b]{self.args.name}[/b] stage in [green]dvc.yaml[/]", f"Creating [b]{self.args.name}[/b] stage in [green]dvc.yaml[/]",
@@ -120,7 +118,6 @@ class CmdExperimentsInit(CmdBase):
def add_parser(experiments_subparsers, parent_parser): def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_INIT_HELP = "Quickly setup any project to use experiments." EXPERIMENTS_INIT_HELP = "Quickly setup any project to use experiments."
experiments_init_parser = experiments_subparsers.add_parser( experiments_init_parser = experiments_subparsers.add_parser(
"init", "init",
@@ -167,35 +164,47 @@ def add_parser(experiments_subparsers, parent_parser):
) )
experiments_init_parser.add_argument( experiments_init_parser.add_argument(
"--code", "--code",
help="Path to the source file or directory " help=(
"which your experiments depend" "Path to the source file or directory "
f" (default: {CmdExperimentsInit.CODE})", "which your experiments depend"
f" (default: {CmdExperimentsInit.CODE})"
),
) )
experiments_init_parser.add_argument( experiments_init_parser.add_argument(
"--data", "--data",
help="Path to the data file or directory " help=(
"which your experiments depend" "Path to the data file or directory "
f" (default: {CmdExperimentsInit.DATA})", "which your experiments depend"
f" (default: {CmdExperimentsInit.DATA})"
),
) )
experiments_init_parser.add_argument( experiments_init_parser.add_argument(
"--models", "--models",
help="Path to the model file or directory for your experiments" help=(
f" (default: {CmdExperimentsInit.MODELS})", "Path to the model file or directory for your experiments"
f" (default: {CmdExperimentsInit.MODELS})"
),
) )
experiments_init_parser.add_argument( experiments_init_parser.add_argument(
"--params", "--params",
help="Path to the parameters file for your experiments" help=(
f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})", "Path to the parameters file for your experiments"
f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})"
),
) )
experiments_init_parser.add_argument( experiments_init_parser.add_argument(
"--metrics", "--metrics",
help="Path to the metrics file for your experiments" help=(
f" (default: {CmdExperimentsInit.DEFAULT_METRICS})", "Path to the metrics file for your experiments"
f" (default: {CmdExperimentsInit.DEFAULT_METRICS})"
),
) )
experiments_init_parser.add_argument( experiments_init_parser.add_argument(
"--plots", "--plots",
help="Path to the plots file or directory for your experiments" help=(
f" (default: {CmdExperimentsInit.PLOTS})", "Path to the plots file or directory for your experiments"
f" (default: {CmdExperimentsInit.PLOTS})"
),
) )
experiments_init_parser.add_argument( experiments_init_parser.add_argument(
"--live", "--live",
+3 -9
View File
@@ -11,9 +11,7 @@ logger = logging.getLogger(__name__)
class CmdExperimentsPull(CmdBase): class CmdExperimentsPull(CmdBase):
def raise_error_if_all_disabled(self): def raise_error_if_all_disabled(self):
if not any( if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
[self.args.experiment, self.args.all_commits, self.args.rev]
):
raise InvalidArgumentError( raise InvalidArgumentError(
"Either provide an `experiment` argument, or use the " "Either provide an `experiment` argument, or use the "
"`--rev` or `--all-commits` flag." "`--rev` or `--all-commits` flag."
@@ -74,10 +72,7 @@ def add_parser(experiments_subparsers, parent_parser):
"--no-cache", "--no-cache",
action="store_false", action="store_false",
dest="pull_cache", dest="pull_cache",
help=( help="Do not pull cached outputs for this experiment from DVC remote storage.",
"Do not pull cached outputs for this experiment from DVC remote "
"storage."
),
) )
experiments_pull_parser.add_argument( experiments_pull_parser.add_argument(
"-r", "-r",
@@ -92,8 +87,7 @@ def add_parser(experiments_subparsers, parent_parser):
type=int, type=int,
metavar="<number>", metavar="<number>",
help=( help=(
"Number of jobs to run simultaneously when pulling from DVC " "Number of jobs to run simultaneously when pulling from DVC remote storage."
"remote storage."
), ),
) )
experiments_pull_parser.add_argument( experiments_pull_parser.add_argument(
+7 -14
View File
@@ -12,16 +12,13 @@ logger = logging.getLogger(__name__)
class CmdExperimentsPush(CmdBase): class CmdExperimentsPush(CmdBase):
def raise_error_if_all_disabled(self): def raise_error_if_all_disabled(self):
if not any( if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
[self.args.experiment, self.args.all_commits, self.args.rev]
):
raise InvalidArgumentError( raise InvalidArgumentError(
"Either provide an `experiment` argument, or use the " "Either provide an `experiment` argument, or use the "
"`--rev` or `--all-commits` flag." "`--rev` or `--all-commits` flag."
) )
def run(self): def run(self):
self.raise_error_if_all_disabled() self.raise_error_if_all_disabled()
pushed_exps = self.repo.experiments.push( pushed_exps = self.repo.experiments.push(
@@ -47,8 +44,10 @@ class CmdExperimentsPush(CmdBase):
if not self.args.push_cache: if not self.args.push_cache:
ui.write( ui.write(
"To push cached outputs", "To push cached outputs",
"for this experiment to DVC remote storage," (
"re-run this command without '--no-cache'.", "for this experiment to DVC remote storage,"
"re-run this command without '--no-cache'."
),
) )
return 0 return 0
@@ -76,10 +75,7 @@ def add_parser(experiments_subparsers, parent_parser):
"--no-cache", "--no-cache",
action="store_false", action="store_false",
dest="push_cache", dest="push_cache",
help=( help="Do not push cached outputs for this experiment to DVC remote storage.",
"Do not push cached outputs for this experiment to DVC remote "
"storage."
),
) )
experiments_push_parser.add_argument( experiments_push_parser.add_argument(
"-r", "-r",
@@ -93,10 +89,7 @@ def add_parser(experiments_subparsers, parent_parser):
"--jobs", "--jobs",
type=int, type=int,
metavar="<number>", metavar="<number>",
help=( help="Number of jobs to run simultaneously when pushing to DVC remote storage.",
"Number of jobs to run simultaneously when pushing to DVC remote "
"storage."
),
) )
experiments_push_parser.add_argument( experiments_push_parser.add_argument(
"--run-cache", "--run-cache",
-1
View File
@@ -24,7 +24,6 @@ class CmdExperimentsRemove(CmdBase):
) )
def run(self): def run(self):
self.check_arguments() self.check_arguments()
removed_list = self.repo.experiments.remove( removed_list = self.repo.experiments.remove(
+2 -6
View File
@@ -17,13 +17,10 @@ class CmdExperimentsRun(CmdRepro):
if self.args.checkpoint_resume: if self.args.checkpoint_resume:
if self.args.reset: if self.args.reset:
raise InvalidArgumentError( raise InvalidArgumentError("--reset and --rev are mutually exclusive.")
"--reset and --rev are mutually exclusive."
)
if not (self.args.queue or self.args.tmp_dir): if not (self.args.queue or self.args.tmp_dir):
raise InvalidArgumentError( raise InvalidArgumentError(
"--rev can only be used in conjunction with " "--rev can only be used in conjunction with --queue or --temp."
"--queue or --temp."
) )
if self.args.reset: if self.args.reset:
@@ -51,7 +48,6 @@ class CmdExperimentsRun(CmdRepro):
def add_parser(experiments_subparsers, parent_parser): def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_RUN_HELP = "Run or resume an experiment." EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
experiments_run_parser = experiments_subparsers.add_parser( experiments_run_parser = experiments_subparsers.add_parser(
"run", "run",
-1
View File
@@ -11,7 +11,6 @@ logger = logging.getLogger(__name__)
class CmdExperimentsSave(CmdBase): class CmdExperimentsSave(CmdBase):
def run(self): def run(self):
try: try:
ref = self.repo.experiments.save( ref = self.repo.experiments.save(
name=self.args.name, name=self.args.name,
+7 -22
View File
@@ -84,9 +84,7 @@ def _collect_rows( # noqa: C901
sort_by, metric_names, param_names sort_by, metric_names, param_names
) )
reverse = sort_order == "desc" reverse = sort_order == "desc"
experiments = _sort_exp( experiments = _sort_exp(experiments, sort_path, sort_name, sort_type, reverse)
experiments, sort_path, sort_name, sort_type, reverse
)
new_checkpoint = True new_checkpoint = True
for i, (rev, results) in enumerate(experiments.items()): for i, (rev, results) in enumerate(experiments.items()):
@@ -116,9 +114,7 @@ def _collect_rows( # noqa: C901
typ = "baseline" typ = "baseline"
elif tip: elif tip:
if tip == parent_tip: if tip == parent_tip:
typ = ( typ = "checkpoint_tip" if new_checkpoint else "checkpoint_commit"
"checkpoint_tip" if new_checkpoint else "checkpoint_commit"
)
elif parent_rev == base_rev: elif parent_rev == base_rev:
typ = "checkpoint_base" typ = "checkpoint_base"
else: else:
@@ -135,9 +131,7 @@ def _collect_rows( # noqa: C901
row_dict["Experiment"] = exp.get("name", "") row_dict["Experiment"] = exp.get("name", "")
row_dict["rev"] = name_rev row_dict["rev"] = name_rev
row_dict["typ"] = typ row_dict["typ"] = typ
row_dict["Created"] = format_time( row_dict["Created"] = format_time(exp.get("timestamp"), fill_value, iso)
exp.get("timestamp"), fill_value, iso
)
row_dict["parent"] = parent row_dict["parent"] = parent
row_dict["State"] = state row_dict["State"] = state
row_dict["Executor"] = exp.get("executor", fill_value) row_dict["Executor"] = exp.get("executor", fill_value)
@@ -371,9 +365,7 @@ def show_experiments( # noqa: C901
if not csv: if not csv:
merge_headers = ["Experiment", "rev", "typ", "parent"] merge_headers = ["Experiment", "rev", "typ", "parent"]
td.column("Experiment")[:] = map( td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers))
prepare_exp_id, td.as_dict(merge_headers)
)
td.drop(*merge_headers[1:]) td.drop(*merge_headers[1:])
headers = { headers = {
@@ -410,9 +402,7 @@ def show_experiments( # noqa: C901
cols_to_drop = set() cols_to_drop = set()
if drop is not None: if drop is not None:
cols_to_drop = { cols_to_drop = {col for col in td.keys() if re.match(drop, col)} # noqa: SIM118
col for col in td.keys() if re.match(drop, col) # noqa: SIM118
}
if pcp: if pcp:
cols_to_drop.add("Created") cols_to_drop.add("Created")
td.drop(*cols_to_drop) td.drop(*cols_to_drop)
@@ -491,9 +481,7 @@ class CmdExperimentsShow(CmdBase):
ui.write_json(all_experiments, default=_format_json) ui.write_json(all_experiments, default=_format_json)
else: else:
precision = ( precision = (
self.args.precision or None self.args.precision or None if self.args.csv else DEFAULT_PRECISION
if self.args.csv
else DEFAULT_PRECISION
) )
fill_value = "" if self.args.csv else FILL_VALUE fill_value = "" if self.args.csv else FILL_VALUE
iso = self.args.csv iso = self.args.csv
@@ -582,10 +570,7 @@ def add_parser(experiments_subparsers, parent_parser):
) )
experiments_show_parser.add_argument( experiments_show_parser.add_argument(
"--sort-order", "--sort-order",
help=( help="Sort order to use with --sort-by. Defaults to ascending ('asc').",
"Sort order to use with --sort-by."
" Defaults to ascending ('asc')."
),
choices=("asc", "desc"), choices=("asc", "desc"),
default="asc", default="asc",
) )
+7 -10
View File
@@ -40,14 +40,9 @@ class CmdGC(CmdBase):
elif self.args.all_tags: elif self.args.all_tags:
msg += " and all git tags" msg += " and all git tags"
if self.args.commit_date: if self.args.commit_date:
msg += ( msg += f" and all git commits before date {self.args.commit_date}"
" and all git commits before date "
f"{self.args.commit_date}"
)
if self.args.rev: if self.args.rev:
msg += ( msg += f" and last {self.args.num} commits from {self.args.rev}"
f" and last {self.args.num} commits from {self.args.rev}"
)
if self.args.all_experiments: if self.args.all_experiments:
msg += " and all experiments" msg += " and all experiments"
@@ -198,9 +193,11 @@ def add_parser(subparsers, parent_parser):
dest="repos", dest="repos",
type=str, type=str,
nargs="*", nargs="*",
help="Keep data files required by these projects " help=(
"in addition to the current one. " "Keep data files required by these projects "
"Useful if you share a single cache across repos.", "in addition to the current one. "
"Useful if you share a single cache across repos."
),
metavar="<paths>", metavar="<paths>",
) )
gc_parser.set_defaults(func=CmdGC) gc_parser.set_defaults(func=CmdGC)
+5 -5
View File
@@ -16,9 +16,7 @@ class CmdGet(CmdBaseNoRepo):
from dvc.ui import ui from dvc.ui import ui
try: try:
url = get_url( url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
self.args.path, repo=self.args.url, rev=self.args.rev
)
ui.write(url, force=True) ui.write(url, force=True)
except DvcException: except DvcException:
logger.exception("failed to show URL") logger.exception("failed to show URL")
@@ -85,8 +83,10 @@ def add_parser(subparsers, parent_parser):
get_parser.add_argument( get_parser.add_argument(
"--show-url", "--show-url",
action="store_true", action="store_true",
help="Print the storage location (URL) the target data would be " help=(
"downloaded from, and exit.", "Print the storage location (URL) the target data would be "
"downloaded from, and exit."
),
) )
get_parser.add_argument( get_parser.add_argument(
"-j", "-j",
+4 -2
View File
@@ -90,8 +90,10 @@ def add_parser(subparsers, parent_parser):
"--no-download", "--no-download",
action="store_true", action="store_true",
default=False, default=False,
help="Create .dvc file including target data hash value(s)" help=(
" but do not actually download the file(s).", "Create .dvc file including target data hash value(s)"
" but do not actually download the file(s)."
),
) )
import_parser.add_argument( import_parser.add_argument(
"-j", "-j",
+22 -21
View File
@@ -29,8 +29,10 @@ class CmdImportUrl(CmdBase):
) )
except DvcException: except DvcException:
logger.exception( logger.exception(
"failed to import %s. You could also try downloading " (
"it manually, and adding it with `dvc add`.", "failed to import %s. You could also try downloading "
"it manually, and adding it with `dvc add`."
),
self.args.url, self.args.url,
) )
return 1 return 1
@@ -40,9 +42,7 @@ class CmdImportUrl(CmdBase):
def add_parser(subparsers, parent_parser): def add_parser(subparsers, parent_parser):
from .add import _add_annotating_args from .add import _add_annotating_args
IMPORT_HELP = ( IMPORT_HELP = "Download or copy file from URL and take it under DVC control."
"Download or copy file from URL and take it under DVC control."
)
import_parser = subparsers.add_parser( import_parser = subparsers.add_parser(
"import-url", "import-url",
@@ -53,16 +53,18 @@ def add_parser(subparsers, parent_parser):
) )
import_parser.add_argument( import_parser.add_argument(
"url", "url",
help="Location of the data to download. Supported URLs:\n" help=(
"/absolute/path/to/file/or/dir\n" "Location of the data to download. Supported URLs:\n"
"relative/path/to/file/or/dir\n" "/absolute/path/to/file/or/dir\n"
"C:\\\\path\\to\\file\\or\\dir\n" "relative/path/to/file/or/dir\n"
"https://example.com/path/to/file\n" "C:\\\\path\\to\\file\\or\\dir\n"
"s3://bucket/key/path\n" "https://example.com/path/to/file\n"
"gs://bucket/path/to/file/or/dir\n" "s3://bucket/key/path\n"
"hdfs://example.com/path/to/file\n" "gs://bucket/path/to/file/or/dir\n"
"ssh://example.com/absolute/path/to/file/or/dir\n" "hdfs://example.com/path/to/file\n"
"remote://remote_name/path/to/file/or/dir (see `dvc remote`)", "ssh://example.com/absolute/path/to/file/or/dir\n"
"remote://remote_name/path/to/file/or/dir (see `dvc remote`)"
),
) )
import_parser.add_argument( import_parser.add_argument(
"out", nargs="?", help="Destination path to put files to." "out", nargs="?", help="Destination path to put files to."
@@ -95,8 +97,10 @@ def add_parser(subparsers, parent_parser):
"--no-download", "--no-download",
action="store_true", action="store_true",
default=False, default=False,
help="Create .dvc file including target data hash value(s)" help=(
" but do not actually download the file(s).", "Create .dvc file including target data hash value(s)"
" but do not actually download the file(s)."
),
) )
import_parser.add_argument( import_parser.add_argument(
"-j", "-j",
@@ -112,10 +116,7 @@ def add_parser(subparsers, parent_parser):
"--version-aware", "--version-aware",
action="store_true", action="store_true",
default=False, default=False,
help=( help="Import using cloud versioning. Implied if the URL contains a version ID.",
"Import using cloud versioning. Implied if the URL contains a "
"version ID."
),
) )
_add_annotating_args(import_parser) _add_annotating_args(import_parser)
import_parser.set_defaults(func=CmdImportUrl) import_parser.set_defaults(func=CmdImportUrl)
+4 -4
View File
@@ -75,8 +75,9 @@ def add_parser(subparsers, parent_parser):
"--no-scm", "--no-scm",
action="store_true", action="store_true",
default=False, default=False,
help="Initiate DVC in directory that is " help=(
"not tracked by any SCM tool (e.g. Git).", "Initiate DVC in directory that is not tracked by any SCM tool (e.g. Git)."
),
) )
init_parser.add_argument( init_parser.add_argument(
"-f", "-f",
@@ -84,8 +85,7 @@ def add_parser(subparsers, parent_parser):
action="store_true", action="store_true",
default=False, default=False,
help=( help=(
"Overwrite existing '.dvc/' directory. " "Overwrite existing '.dvc/' directory. This operation removes local cache."
"This operation removes local cache."
), ),
) )
init_parser.add_argument( init_parser.add_argument(
+4 -2
View File
@@ -31,7 +31,9 @@ def add_parser(subparsers, parent_parser):
"--use-pre-commit-tool", "--use-pre-commit-tool",
action="store_true", action="store_true",
default=False, default=False,
help="Install DVC hooks using pre-commit " help=(
"(https://pre-commit.com) if it is installed.", "Install DVC hooks using pre-commit "
"(https://pre-commit.com) if it is installed."
),
) )
install_parser.set_defaults(func=CmdInstall) install_parser.set_defaults(func=CmdInstall)
+9 -28
View File
@@ -18,7 +18,6 @@ class MachineDisabledError(ConfigError):
class CmdMachineConfig(CmdConfig): class CmdMachineConfig(CmdConfig):
def __init__(self, args): def __init__(self, args):
super().__init__(args) super().__init__(args)
if not self.config["feature"].get("machine", False): if not self.config["feature"].get("machine", False):
raise MachineDisabledError raise MachineDisabledError
@@ -163,14 +162,11 @@ class CmdMachineRename(CmdBase):
all_config = self.config.load_config_to_level(None) all_config = self.config.load_config_to_level(None)
if self.args.new in all_config.get("machine", {}): if self.args.new in all_config.get("machine", {}):
raise ConfigError( raise ConfigError(
"Rename failed. Machine '{}' already exists.".format( f"Rename failed. Machine '{self.args.new}' already exists."
self.args.new
)
) )
ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.") ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.")
def run(self): def run(self):
self._check_before_rename() self._check_before_rename()
with self.config.edit(self.args.level) as conf: with self.config.edit(self.args.level) as conf:
@@ -209,9 +205,7 @@ class CmdMachineDefault(CmdMachineConfig):
if self.args.unset: if self.args.unset:
conf["core"].pop("machine", None) conf["core"].pop("machine", None)
else: else:
merged_conf = self.config.load_config_to_level( merged_conf = self.config.load_config_to_level(self.args.level)
self.args.level
)
if ( if (
self.args.name in conf["machine"] self.args.name in conf["machine"]
or self.args.name in merged_conf["machine"] or self.args.name in merged_conf["machine"]
@@ -219,8 +213,7 @@ class CmdMachineDefault(CmdMachineConfig):
conf["core"]["machine"] = self.args.name conf["core"]["machine"] = self.args.name
else: else:
raise ConfigError( raise ConfigError(
"default machine must be present in machine " "default machine must be present in machine list."
"list."
) )
return 0 return 0
@@ -251,7 +244,6 @@ class CmdMachineStatus(CmdBase):
all_status: List[Dict], all_status: List[Dict],
td: TabularData, td: TabularData,
): ):
if not all_status: if not all_status:
row = [ row = [
name, name,
@@ -343,8 +335,7 @@ def add_parser(subparsers, parent_parser):
"cloud", "cloud",
help="Machine cloud. See full list of supported clouds at {}".format( help="Machine cloud. See full list of supported clouds at {}".format(
format_link( format_link(
"https://github.com/iterative/" "https://github.com/iterative/terraform-provider-iterative#machine"
"terraform-provider-iterative#machine"
) )
), ),
) )
@@ -372,9 +363,7 @@ def add_parser(subparsers, parent_parser):
help=machine_DEFAULT_HELP, help=machine_DEFAULT_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
) )
machine_default_parser.add_argument( machine_default_parser.add_argument("name", nargs="?", help="Name of the machine")
"name", nargs="?", help="Name of the machine"
)
machine_default_parser.add_argument( machine_default_parser.add_argument(
"-u", "-u",
"--unset", "--unset",
@@ -414,9 +403,7 @@ def add_parser(subparsers, parent_parser):
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
) )
machine_modify_parser.add_argument("name", help="Name of the machine") machine_modify_parser.add_argument("name", help="Name of the machine")
machine_modify_parser.add_argument( machine_modify_parser.add_argument("option", help="Name of the option to modify.")
"option", help="Name of the option to modify."
)
machine_modify_parser.add_argument( machine_modify_parser.add_argument(
"value", nargs="?", help="(optional) Value of the option." "value", nargs="?", help="(optional) Value of the option."
) )
@@ -449,9 +436,7 @@ def add_parser(subparsers, parent_parser):
help=machine_REMOVE_HELP, help=machine_REMOVE_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
) )
machine_remove_parser.add_argument( machine_remove_parser.add_argument("name", help="Name of the machine to remove.")
"name", help="Name of the machine to remove."
)
machine_remove_parser.set_defaults(func=CmdMachineRemove) machine_remove_parser.set_defaults(func=CmdMachineRemove)
machine_CREATE_HELP = "Create and start a machine instance." machine_CREATE_HELP = "Create and start a machine instance."
@@ -462,14 +447,10 @@ def add_parser(subparsers, parent_parser):
help=machine_CREATE_HELP, help=machine_CREATE_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
) )
machine_create_parser.add_argument( machine_create_parser.add_argument("name", help="Name of the machine to create.")
"name", help="Name of the machine to create."
)
machine_create_parser.set_defaults(func=CmdMachineCreate) machine_create_parser.set_defaults(func=CmdMachineCreate)
machine_STATUS_HELP = ( machine_STATUS_HELP = "List the status of running instances for one/all machines."
"List the status of running instances for one/all machines."
)
machine_status_parser = machine_subparsers.add_parser( machine_status_parser = machine_subparsers.add_parser(
"status", "status",
parents=[parent_parser], parents=[parent_parser],
+2 -6
View File
@@ -14,9 +14,7 @@ class CmdMove(CmdBase):
try: try:
self.repo.move(self.args.src, self.args.dst) self.repo.move(self.args.src, self.args.dst)
except DvcException: except DvcException:
msg = "failed to move '{}' -> '{}'".format( msg = f"failed to move '{self.args.src}' -> '{self.args.dst}'"
self.args.src, self.args.dst
)
logger.exception(msg) logger.exception(msg)
return 1 return 1
return 0 return 0
@@ -40,7 +38,5 @@ def add_parser(subparsers, parent_parser):
move_parser.add_argument( move_parser.add_argument(
"src", help="Source path to a data file or directory." "src", help="Source path to a data file or directory."
).complete = completion.FILE ).complete = completion.FILE
move_parser.add_argument( move_parser.add_argument("dst", help="Destination path.").complete = completion.FILE
"dst", help="Destination path."
).complete = completion.FILE
move_parser.set_defaults(func=CmdMove) move_parser.set_defaults(func=CmdMove)
+1 -1
View File
@@ -79,7 +79,7 @@ def add_parser(subparsers, parent_parser):
params_diff_parser.add_argument( params_diff_parser.add_argument(
"b_rev", "b_rev",
nargs="?", nargs="?",
help=("New Git commit to compare (defaults to the current workspace)"), help="New Git commit to compare (defaults to the current workspace)",
) )
params_diff_parser.add_argument( params_diff_parser.add_argument(
"--targets", "--targets",
+13 -32
View File
@@ -18,9 +18,7 @@ logger = logging.getLogger(__name__)
def _show_json(renderers, split=False): def _show_json(renderers, split=False):
from dvc.render.convert import to_json from dvc.render.convert import to_json
result = { result = {renderer.name: to_json(renderer, split) for renderer in renderers}
renderer.name: to_json(renderer, split) for renderer in renderers
}
ui.write_json(result) ui.write_json(result)
@@ -90,10 +88,8 @@ class CmdPlots(CmdBase):
) )
if html_template_path and not os.path.isabs(html_template_path): if html_template_path and not os.path.isabs(html_template_path):
assert self.repo.dvc_dir assert self.repo.dvc_dir
html_template_path = os.path.join( html_template_path = os.path.join(self.repo.dvc_dir, html_template_path)
self.repo.dvc_dir, html_template_path return html_template_path # noqa: RET504
)
return html_template_path
def run(self) -> int: # noqa: C901 def run(self) -> int: # noqa: C901
from pathlib import Path from pathlib import Path
@@ -106,14 +102,11 @@ class CmdPlots(CmdBase):
logger.error("please specify a target for `--show-vega`") logger.error("please specify a target for `--show-vega`")
return 1 return 1
if len(self.args.targets) > 1: if len(self.args.targets) > 1:
logger.error( logger.error("you can only specify one target for `--show-vega`")
"you can only specify one target for `--show-vega`"
)
return 1 return 1
if self.args.json: if self.args.json:
logger.error( logger.error(
"'--show-vega' and '--json' are mutually exclusive " "'--show-vega' and '--json' are mutually exclusive options."
"options."
) )
return 1 return 1
@@ -126,17 +119,14 @@ class CmdPlots(CmdBase):
if not plots_data: if not plots_data:
ui.error_write( ui.error_write(
"No plots were loaded, " "No plots were loaded, visualization file will not be created."
"visualization file will not be created."
) )
out: str = self.args.out or self.repo.config.get("plots", {}).get( out: str = self.args.out or self.repo.config.get("plots", {}).get(
"out_dir", "dvc_plots" "out_dir", "dvc_plots"
) )
renderers_out = ( renderers_out = out if self.args.json else os.path.join(out, "static")
out if self.args.json else os.path.join(out, "static")
)
renderers = match_defs_renderers( renderers = match_defs_renderers(
data=plots_data, data=plots_data,
@@ -249,8 +239,7 @@ def add_parser(subparsers, parent_parser):
fix_subparsers(plots_subparsers) fix_subparsers(plots_subparsers)
SHOW_HELP = ( SHOW_HELP = (
"Generate plots from target files or from `plots`" "Generate plots from target files or from `plots` definitions in `dvc.yaml`."
" definitions in `dvc.yaml`."
) )
plots_show_parser = plots_subparsers.add_parser( plots_show_parser = plots_subparsers.add_parser(
"show", "show",
@@ -273,8 +262,7 @@ def add_parser(subparsers, parent_parser):
plots_show_parser.set_defaults(func=CmdPlotsShow) plots_show_parser.set_defaults(func=CmdPlotsShow)
PLOTS_DIFF_HELP = ( PLOTS_DIFF_HELP = (
"Show multiple versions of a plot by overlaying them " "Show multiple versions of a plot by overlaying them in a single image."
"in a single image."
) )
plots_diff_parser = plots_subparsers.add_parser( plots_diff_parser = plots_subparsers.add_parser(
"diff", "diff",
@@ -332,9 +320,7 @@ def add_parser(subparsers, parent_parser):
) )
plots_modify_parser.set_defaults(func=CmdPlotsModify) plots_modify_parser.set_defaults(func=CmdPlotsModify)
TEMPLATES_HELP = ( TEMPLATES_HELP = "List built-in plots templates or show JSON specification for one."
"List built-in plots templates or show JSON specification for one."
)
plots_templates_parser = plots_subparsers.add_parser( plots_templates_parser = plots_subparsers.add_parser(
"templates", "templates",
parents=[parent_parser], parents=[parent_parser],
@@ -360,11 +346,8 @@ def _add_props_arguments(parser):
"--template", "--template",
nargs="?", nargs="?",
default=None, default=None,
help=( help="Special JSON or HTML schema file to inject with the data. See {}".format(
"Special JSON or HTML schema file to inject with the data. " format_link("https://man.dvc.org/plots#plot-templates")
"See {}".format(
format_link("https://man.dvc.org/plots#plot-templates")
)
), ),
metavar="<path>", metavar="<path>",
).complete = completion.FILE ).complete = completion.FILE
@@ -381,9 +364,7 @@ def _add_props_arguments(parser):
default=None, # Use default None to distinguish when it's not used default=None, # Use default None to distinguish when it's not used
help="Provided CSV or TSV datafile does not have a header.", help="Provided CSV or TSV datafile does not have a header.",
) )
parser.add_argument( parser.add_argument("--title", default=None, metavar="<text>", help="Plot title.")
"--title", default=None, metavar="<text>", help="Plot title."
)
parser.add_argument( parser.add_argument(
"--x-label", default=None, help="X axis label", metavar="<text>" "--x-label", default=None, help="X axis label", metavar="<text>"
) )
+1 -2
View File
@@ -20,8 +20,7 @@ class CmdQueueKill(CmdBase):
def add_parser(queue_subparsers, parent_parser): def add_parser(queue_subparsers, parent_parser):
QUEUE_KILL_HELP = ( QUEUE_KILL_HELP = (
"Gracefully interrupt running experiment queue tasks " "Gracefully interrupt running experiment queue tasks (equivalent to Ctrl-C)"
"(equivalent to Ctrl-C)"
) )
queue_kill_parser = queue_subparsers.add_parser( queue_kill_parser = queue_subparsers.add_parser(
"kill", "kill",
+1 -3
View File
@@ -34,9 +34,7 @@ def add_parser(queue_subparsers, parent_parser):
queue_logs_parser.add_argument( queue_logs_parser.add_argument(
"-e", "-e",
"--encoding", "--encoding",
help=( help="Text encoding for log output. Defaults to system locale encoding.",
"Text encoding for log output. Defaults to system locale encoding."
),
metavar="<encoding>", metavar="<encoding>",
) )
queue_logs_parser.add_argument( queue_logs_parser.add_argument(
-2
View File
@@ -28,7 +28,6 @@ class CmdQueueRemove(CmdBase):
) )
def run(self): def run(self):
self.check_arguments() self.check_arguments()
if self.args.all: if self.args.all:
@@ -57,7 +56,6 @@ class CmdQueueRemove(CmdBase):
def add_parser(queue_subparsers, parent_parser): def add_parser(queue_subparsers, parent_parser):
QUEUE_REMOVE_HELP = "Remove queued and completed tasks from the queue." QUEUE_REMOVE_HELP = "Remove queued and completed tasks from the queue."
queue_remove_parser = queue_subparsers.add_parser( queue_remove_parser = queue_subparsers.add_parser(
"remove", "remove",
+3 -11
View File
@@ -12,20 +12,15 @@ class CmdQueueStart(CmdBase):
"""Start exp queue workers.""" """Start exp queue workers."""
def run(self): def run(self):
started = self.repo.experiments.celery_queue.start_workers( started = self.repo.experiments.celery_queue.start_workers(self.args.jobs)
self.args.jobs
)
suffix = "s" if started > 1 else "" suffix = "s" if started > 1 else ""
ui.write( ui.write(f"Started '{started}' new experiments task queue worker{suffix}.")
f"Started '{started}' new experiments task queue worker{suffix}."
)
return 0 return 0
def add_parser(queue_subparsers, parent_parser): def add_parser(queue_subparsers, parent_parser):
QUEUE_START_HELP = "Start the experiments task queue worker." QUEUE_START_HELP = "Start the experiments task queue worker."
queue_start_parser = queue_subparsers.add_parser( queue_start_parser = queue_subparsers.add_parser(
"start", "start",
@@ -39,10 +34,7 @@ def add_parser(queue_subparsers, parent_parser):
"--jobs", "--jobs",
type=int, type=int,
default=1, default=1,
help=( help="Maximum number of concurrent queue workers to start. Defaults to 1.",
"Maximum number of concurrent queue workers to start. "
"Defaults to 1."
),
metavar="<number>", metavar="<number>",
) )
queue_start_parser.set_defaults(func=CmdQueueStart) queue_start_parser.set_defaults(func=CmdQueueStart)
+2 -6
View File
@@ -39,9 +39,7 @@ class CmdQueueStatus(CmdBase):
ui.write() ui.write()
worker_status = self.repo.experiments.celery_queue.worker_status() worker_status = self.repo.experiments.celery_queue.worker_status()
active_count = len( active_count = len([name for name, task in worker_status.items() if task])
[name for name, task in worker_status.items() if task]
)
idle_count = len(worker_status) - active_count idle_count = len(worker_status) - active_count
ui.write(f"Worker status: {active_count} active, {idle_count} idle") ui.write(f"Worker status: {active_count} active, {idle_count} idle")
@@ -50,9 +48,7 @@ class CmdQueueStatus(CmdBase):
def add_parser(queue_subparsers, parent_parser): def add_parser(queue_subparsers, parent_parser):
QUEUE_STATUS_HELP = ( QUEUE_STATUS_HELP = "Show the status of experiments queue tasks and workers."
"Show the status of experiments queue tasks and workers."
)
queue_status_parser = queue_subparsers.add_parser( queue_status_parser = queue_subparsers.add_parser(
"status", "status",
parents=[parent_parser], parents=[parent_parser],
-1
View File
@@ -26,7 +26,6 @@ class CmdQueueStop(CmdBase):
def add_parser(queue_subparsers, parent_parser): def add_parser(queue_subparsers, parent_parser):
QUEUE_STOP_HELP = "Stop all experiments task queue workers." QUEUE_STOP_HELP = "Stop all experiments task queue workers."
queue_stop_parser = queue_subparsers.add_parser( queue_stop_parser = queue_subparsers.add_parser(
"stop", "stop",
+4 -12
View File
@@ -95,9 +95,7 @@ class CmdRemoteDefault(CmdRemote):
if self.args.unset: if self.args.unset:
conf["core"].pop("remote", None) conf["core"].pop("remote", None)
else: else:
merged_conf = self.config.load_config_to_level( merged_conf = self.config.load_config_to_level(self.args.level)
self.args.level
)
if ( if (
self.args.name in conf["remote"] self.args.name in conf["remote"]
or self.args.name in merged_conf["remote"] or self.args.name in merged_conf["remote"]
@@ -208,9 +206,7 @@ def add_parser(subparsers, parent_parser):
help=REMOTE_DEFAULT_HELP, help=REMOTE_DEFAULT_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
) )
remote_default_parser.add_argument( remote_default_parser.add_argument("name", nargs="?", help="Name of the remote")
"name", nargs="?", help="Name of the remote"
)
remote_default_parser.add_argument( remote_default_parser.add_argument(
"-u", "-u",
"--unset", "--unset",
@@ -229,9 +225,7 @@ def add_parser(subparsers, parent_parser):
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
) )
remote_modify_parser.add_argument("name", help="Name of the remote") remote_modify_parser.add_argument("name", help="Name of the remote")
remote_modify_parser.add_argument( remote_modify_parser.add_argument("option", help="Name of the option to modify.")
"option", help="Name of the option to modify."
)
remote_modify_parser.add_argument( remote_modify_parser.add_argument(
"value", nargs="?", help="(optional) Value of the option." "value", nargs="?", help="(optional) Value of the option."
) )
@@ -262,9 +256,7 @@ def add_parser(subparsers, parent_parser):
help=REMOTE_REMOVE_HELP, help=REMOTE_REMOVE_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
) )
remote_remove_parser.add_argument( remote_remove_parser.add_argument("name", help="Name of the remote to remove.")
"name", help="Name of the remote to remove."
)
remote_remove_parser.set_defaults(func=CmdRemoteRemove) remote_remove_parser.set_defaults(func=CmdRemoteRemove)
REMOTE_RENAME_HELP = "Rename a DVC remote" REMOTE_RENAME_HELP = "Rename a DVC remote"
remote_rename_parser = remote_subparsers.add_parser( remote_rename_parser = remote_subparsers.add_parser(
+1 -2
View File
@@ -22,8 +22,7 @@ class CmdRemove(CmdBase):
def add_parser(subparsers, parent_parser): def add_parser(subparsers, parent_parser):
REMOVE_HELP = ( REMOVE_HELP = (
"Remove stages from dvc.yaml and/or" "Remove stages from dvc.yaml and/or stop tracking files or directories."
" stop tracking files or directories."
) )
remove_parser = subparsers.add_parser( remove_parser = subparsers.add_parser(
"remove", "remove",
+13 -18
View File
@@ -10,9 +10,7 @@ class CmdRepro(CmdBase):
def run(self): def run(self):
from dvc.ui import ui from dvc.ui import ui
stages = self.repo.reproduce( stages = self.repo.reproduce(**self._common_kwargs, **self._repro_kwargs)
**self._common_kwargs, **self._repro_kwargs
)
if len(stages) == 0: if len(stages) == 0:
ui.write(CmdDataStatus.UP_TO_DATE_MSG) ui.write(CmdDataStatus.UP_TO_DATE_MSG)
else: else:
@@ -55,16 +53,14 @@ def add_arguments(repro_parser):
repro_parser.add_argument( repro_parser.add_argument(
"targets", "targets",
nargs="*", nargs="*",
help=( help="""\
"""\
Stages to reproduce. 'dvc.yaml' by default. Stages to reproduce. 'dvc.yaml' by default.
The targets can be path to a dvc.yaml file or `.dvc` file, The targets can be path to a dvc.yaml file or `.dvc` file,
or a stage name from dvc.yaml file from or a stage name from dvc.yaml file from
current working directory. To run a stage from dvc.yaml current working directory. To run a stage from dvc.yaml
from other directories, the target must be a path followed by colon `:` from other directories, the target must be a path followed by colon `:`
and then the stage name name. and then the stage name name.
""" """,
),
).complete = completion.DVCFILES_AND_STAGE ).complete = completion.DVCFILES_AND_STAGE
repro_parser.add_argument( repro_parser.add_argument(
"-f", "-f",
@@ -85,16 +81,14 @@ and then the stage name name.
"--single-item", "--single-item",
action="store_true", action="store_true",
default=False, default=False,
help="Reproduce only single data item without recursive dependencies " help="Reproduce only single data item without recursive dependencies check.",
"check.",
) )
repro_parser.add_argument( repro_parser.add_argument(
"-p", "-p",
"--pipeline", "--pipeline",
action="store_true", action="store_true",
default=False, default=False,
help="Reproduce the whole pipeline that the specified targets " help="Reproduce the whole pipeline that the specified targets belong to.",
"belong to.",
) )
repro_parser.add_argument( repro_parser.add_argument(
"-P", "-P",
@@ -127,8 +121,10 @@ and then the stage name name.
"--force-downstream", "--force-downstream",
action="store_true", action="store_true",
default=False, default=False,
help="Reproduce all descendants of a changed stage even if their " help=(
"direct dependencies didn't change.", "Reproduce all descendants of a changed stage even if their "
"direct dependencies didn't change."
),
) )
repro_parser.add_argument( repro_parser.add_argument(
"--pull", "--pull",
@@ -143,15 +139,14 @@ and then the stage name name.
"--dry", "--dry",
action="store_true", action="store_true",
default=False, default=False,
help="Only print the commands that would be executed without " help=(
"actually executing.", "Only print the commands that would be executed without actually executing."
),
) )
def add_parser(subparsers, parent_parser): def add_parser(subparsers, parent_parser):
REPRO_HELP = ( REPRO_HELP = "Reproduce complete or partial pipelines by executing their stages."
"Reproduce complete or partial pipelines by executing their stages."
)
repro_parser = subparsers.add_parser( repro_parser = subparsers.add_parser(
"repro", "repro",
parents=[parent_parser], parents=[parent_parser],
+3 -7
View File
@@ -39,7 +39,7 @@ class CmdRun(CmdBase):
{ {
"cmd": parse_cmd(self.args.command), "cmd": parse_cmd(self.args.command),
"fname": kwargs.pop("file"), "fname": kwargs.pop("file"),
"no_exec": (self.args.no_exec or bool(self.args.checkpoints)), "no_exec": self.args.no_exec or bool(self.args.checkpoints),
"run_cache": not kwargs.pop("no_run_cache"), "run_cache": not kwargs.pop("no_run_cache"),
} }
) )
@@ -55,9 +55,7 @@ class CmdRun(CmdBase):
def add_parser(subparsers, parent_parser): def add_parser(subparsers, parent_parser):
from dvc.commands.stage import _add_common_args from dvc.commands.stage import _add_common_args
RUN_HELP = ( RUN_HELP = "Generate a dvc.yaml file from a command and execute the command."
"Generate a dvc.yaml file from a command and execute the command."
)
run_parser = subparsers.add_parser( run_parser = subparsers.add_parser(
"run", "run",
parents=[parent_parser], parents=[parent_parser],
@@ -66,9 +64,7 @@ def add_parser(subparsers, parent_parser):
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
) )
run_parser.add_argument("-n", "--name", help="Stage name.") run_parser.add_argument("-n", "--name", help="Stage name.")
run_parser.add_argument( run_parser.add_argument("--file", metavar="<filename>", help=argparse.SUPPRESS)
"--file", metavar="<filename>", help=argparse.SUPPRESS
)
run_parser.add_argument( run_parser.add_argument(
"--single-stage", "--single-stage",
action="store_true", action="store_true",
+15 -17
View File
@@ -44,9 +44,7 @@ def generate_description(stage: "Stage") -> str:
return "; ".join(desc) return "; ".join(desc)
def prepare_description( def prepare_description(stage: "Stage", max_length: int = MAX_TEXT_LENGTH) -> str:
stage: "Stage", max_length: int = MAX_TEXT_LENGTH
) -> str:
desc = stage.short_description() or generate_description(stage) desc = stage.short_description() or generate_description(stage)
return truncate_text(desc, max_length) return truncate_text(desc, max_length)
@@ -57,9 +55,9 @@ def prepare_stages_data(
max_length: int = MAX_TEXT_LENGTH, max_length: int = MAX_TEXT_LENGTH,
) -> Dict[str, str]: ) -> Dict[str, str]:
return { return {
stage.addressing: prepare_description(stage, max_length=max_length) stage.addressing: (
if description prepare_description(stage, max_length=max_length) if description else ""
else "" )
for stage in stages for stage in stages
} }
@@ -75,9 +73,7 @@ class CmdStageList(CmdBase):
# removing duplicates while maintaining order # removing duplicates while maintaining order
collected = chain.from_iterable( collected = chain.from_iterable(
self.repo.stage.collect( self.repo.stage.collect(target=target, recursive=self.args.recursive)
target=target, recursive=self.args.recursive
)
for target in self.args.targets for target in self.args.targets
) )
return dict.fromkeys(collected).keys() return dict.fromkeys(collected).keys()
@@ -173,8 +169,7 @@ def _add_common_args(parser):
"--outs-no-cache", "--outs-no-cache",
action="append", action="append",
default=[], default=[],
help="Declare output file or directory " help="Declare output file or directory (do not put into DVC cache).",
"(do not put into DVC cache).",
metavar="<filename>", metavar="<filename>",
).complete = completion.FILE ).complete = completion.FILE
parser.add_argument( parser.add_argument(
@@ -182,8 +177,10 @@ def _add_common_args(parser):
"--checkpoints", "--checkpoints",
action="append", action="append",
default=[], default=[],
help="Declare checkpoint output file or directory for 'dvc exp run'. " help=(
"Not compatible with 'dvc repro'.", "Declare checkpoint output file or directory for 'dvc exp run'. "
"Not compatible with 'dvc repro'."
),
metavar="<filename>", metavar="<filename>",
).complete = completion.FILE ).complete = completion.FILE
parser.add_argument( parser.add_argument(
@@ -196,16 +193,17 @@ def _add_common_args(parser):
"--outs-persist", "--outs-persist",
action="append", action="append",
default=[], default=[],
help="Declare output file or directory that will not be " help="Declare output file or directory that will not be removed upon repro.",
"removed upon repro.",
metavar="<filename>", metavar="<filename>",
) )
parser.add_argument( parser.add_argument(
"--outs-persist-no-cache", "--outs-persist-no-cache",
action="append", action="append",
default=[], default=[],
help="Declare output file or directory that will not be " help=(
"removed upon repro (do not put into DVC cache).", "Declare output file or directory that will not be "
"removed upon repro (do not put into DVC cache)."
),
metavar="<filename>", metavar="<filename>",
) )
parser.add_argument( parser.add_argument(
+1 -3
View File
@@ -81,9 +81,7 @@ class CmdDataStatus(CmdDataBase):
if not self.repo.index.stages: if not self.repo.index.stages:
ui.write(self.EMPTY_PROJECT_MSG) ui.write(self.EMPTY_PROJECT_MSG)
elif self.args.cloud or self.args.remote: elif self.args.cloud or self.args.remote:
remote = self.args.remote or self.repo.config["core"].get( remote = self.args.remote or self.repo.config["core"].get("remote")
"remote"
)
ui.write(self.IN_SYNC_MSG.format(remote=remote)) ui.write(self.IN_SYNC_MSG.format(remote=remote))
else: else:
ui.write(self.UP_TO_DATE_MSG) ui.write(self.UP_TO_DATE_MSG)
+4 -2
View File
@@ -61,8 +61,10 @@ def add_parser(subparsers, parent_parser):
"--no-download", "--no-download",
action="store_true", action="store_true",
default=False, default=False,
help="Update .dvc file git revision/hash value(s)" help=(
" but do not download the file(s).", "Update .dvc file git revision/hash value(s)"
" but do not download the file(s)."
),
) )
update_parser.add_argument( update_parser.add_argument(
"--to-remote", "--to-remote",
+1 -3
View File
@@ -21,9 +21,7 @@ class CmdVersion(CmdBaseNoRepo):
def add_parser(subparsers, parent_parser): def add_parser(subparsers, parent_parser):
VERSION_HELP = ( VERSION_HELP = "Display the DVC version and system/environment information."
"Display the DVC version and system/environment information."
)
version_parser = subparsers.add_parser( version_parser = subparsers.add_parser(
"version", "version",
parents=[parent_parser], parents=[parent_parser],
+9 -26
View File
@@ -64,9 +64,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
def keys(self) -> List[str]: def keys(self) -> List[str]:
return self._keys return self._keys
def _iter_col_row( def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[Tuple["CellT", Column]]:
self, row: Sequence["CellT"]
) -> Iterator[Tuple["CellT", Column]]:
for val, col in zip_longest(row, self.columns): for val, col in zip_longest(row, self.columns):
if col is None: if col is None:
break break
@@ -103,9 +101,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
... ...
@overload @overload
def __setitem__( def __setitem__(self, item: slice, value: Iterable[Sequence["CellT"]]) -> None:
self, item: slice, value: Iterable[Sequence["CellT"]]
) -> None:
... ...
def __setitem__(self, item, value) -> None: def __setitem__(self, item, value) -> None:
@@ -113,8 +109,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
if isinstance(item, slice): if isinstance(item, slice):
n = len(self.columns) n = len(self.columns)
normalized_rows = ( normalized_rows = (
chain(val, repeat(self._fill_value, n - len(val))) chain(val, repeat(self._fill_value, n - len(val))) for val in value
for val in value
) )
# we need to transpose those rows into columnar format # we need to transpose those rows into columnar format
# as we work in terms of column-based arrays # as we work in terms of column-based arrays
@@ -212,9 +207,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
self, cols: Optional[Iterable[str]] = None self, cols: Optional[Iterable[str]] = None
) -> Iterable[Dict[str, "CellT"]]: ) -> Iterable[Dict[str, "CellT"]]:
keys = self.keys() if cols is None else set(cols) keys = self.keys() if cols is None else set(cols)
return [ return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
{k: self._columns[k][i] for k in keys} for i in range(len(self))
]
def dropna( # noqa: C901 def dropna( # noqa: C901
self, self,
@@ -224,13 +217,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
): ):
if axis not in ["rows", "cols"]: if axis not in ["rows", "cols"]:
raise ValueError( raise ValueError(
f"Invalid 'axis' value {axis}." f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']"
"Choose one of ['rows', 'cols']"
) )
if how not in ["any", "all"]: if how not in ["any", "all"]:
raise ValueError( raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']")
f"Invalid 'how' value {how}. Choose one of ['any', 'all']"
)
match_line: Set = set() match_line: Set = set()
match_any = True match_any = True
@@ -258,11 +248,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
if axis == "rows": if axis == "rows":
for name in self.keys(): # noqa: SIM118 for name in self.keys(): # noqa: SIM118
self._columns[name] = Column( self._columns[name] = Column(
[ [x for n, x in enumerate(self._columns[name]) if n not in to_drop]
x
for n, x in enumerate(self._columns[name])
if n not in to_drop
]
) )
else: else:
self.drop(*to_drop) self.drop(*to_drop)
@@ -275,8 +261,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
): ):
if axis not in ["rows", "cols"]: if axis not in ["rows", "cols"]:
raise ValueError( raise ValueError(
f"Invalid 'axis' value {axis}." f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']"
"Choose one of ['rows', 'cols']"
) )
if axis == "cols": if axis == "cols":
@@ -362,9 +347,7 @@ def diff_table(
for item, change in sorted(diff_in_file.items()): for item, change in sorted(diff_in_file.items()):
old_value = with_value(change.get("old"), fill_value) old_value = with_value(change.get("old"), fill_value)
new_value = with_value(change.get("new"), fill_value) new_value = with_value(change.get("new"), fill_value)
diff_value = with_value( diff_value = with_value(change.get("diff", on_empty_diff), fill_value)
change.get("diff", on_empty_diff), fill_value
)
td.append( td.append(
[ [
fname, fname,
+2 -5
View File
@@ -153,9 +153,7 @@ class Config(dict):
with open(config_file, "w+", encoding="utf-8"): with open(config_file, "w+", encoding="utf-8"):
return Config(dvc_dir) return Config(dvc_dir)
def load( def load(self, validate: bool = True, config: Optional["DictStrAny"] = None):
self, validate: bool = True, config: Optional["DictStrAny"] = None
):
"""Loads config from all the config files. """Loads config from all the config files.
Raises: Raises:
@@ -376,6 +374,5 @@ def merge(into, update):
def _lower_keys(data): def _lower_keys(data):
return { return {
k.lower(): _lower_keys(v) if isinstance(v, dict) else v k.lower(): _lower_keys(v) if isinstance(v, dict) else v for k, v in data.items()
for k, v in data.items()
} }
+3 -16
View File
@@ -2,16 +2,7 @@ import os
from urllib.parse import urlparse from urllib.parse import urlparse
from funcy import walk_values from funcy import walk_values
from voluptuous import ( from voluptuous import All, Any, Coerce, Invalid, Lower, Optional, Range, Schema
All,
Any,
Coerce,
Invalid,
Lower,
Optional,
Range,
Schema,
)
Bool = All( Bool = All(
Lower, Lower,
@@ -34,9 +25,7 @@ def supported_cache_type(types):
unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"} unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
if unsupported: if unsupported:
raise Invalid( raise Invalid("Unsupported cache type(s): {}".format(", ".join(unsupported)))
"Unsupported cache type(s): {}".format(", ".join(unsupported))
)
return types return types
@@ -258,9 +247,7 @@ SCHEMA = {
"machine": { "machine": {
str: { str: {
"cloud": All(Lower, Choices("aws", "azure")), "cloud": All(Lower, Choices("aws", "azure")),
"region": All( "region": All(Lower, Choices("us-west", "us-east", "eu-west", "eu-north")),
Lower, Choices("us-west", "us-east", "eu-west", "eu-north")
),
"image": str, "image": str,
"spot": Bool, "spot": Bool,
"spot_price": Coerce(float), "spot_price": Coerce(float),
+1 -3
View File
@@ -40,9 +40,7 @@ def _spawn_windows(cmd, env):
startupinfo = STARTUPINFO() startupinfo = STARTUPINFO()
startupinfo.dwFlags |= STARTF_USESHOWWINDOW startupinfo.dwFlags |= STARTF_USESHOWWINDOW
_popen( _popen(cmd, env=env, creationflags=creationflags, startupinfo=startupinfo)
cmd, env=env, creationflags=creationflags, startupinfo=startupinfo
)
def _spawn_posix(cmd, env): def _spawn_posix(cmd, env):
+1 -3
View File
@@ -272,8 +272,6 @@ def draw(vertices, edges):
vertex.view.h, vertex.view.h,
) )
canvas.text( canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
)
return canvas.draw() return canvas.draw()
+4 -2
View File
@@ -111,8 +111,10 @@ class DataCloud:
for hash_info in status.missing for hash_info in status.missing
) )
logger.warning( logger.warning(
"Some of the cache files do not exist neither locally " (
"nor on remote. Missing cache files:\n%s", "Some of the cache files do not exist neither locally "
"nor on remote. Missing cache files:\n%s"
),
missing_desc, missing_desc,
) )
+1 -3
View File
@@ -90,6 +90,4 @@ def _merge_params(s_list):
def loads_params(stage, s_list): def loads_params(stage, s_list):
d = _merge_params(s_list) d = _merge_params(s_list)
return [ return [ParamsDependency(stage, path, params) for path, params in d.items()]
ParamsDependency(stage, path, params) for path, params in d.items()
]
+2 -6
View File
@@ -47,9 +47,7 @@ class Dependency(Output):
self.def_path = self.fs.path.version_path( self.def_path = self.fs.path.version_path(
self.def_path, self.meta.version_id self.def_path, self.meta.version_id
) )
self.fs_path = self.fs.path.version_path( self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
self.fs_path, self.meta.version_id
)
def download(self, to, jobs=None): def download(self, to, jobs=None):
fs_download(self.fs, self.fs_path, to, jobs=jobs) fs_download(self.fs, self.fs_path, to, jobs=jobs)
@@ -57,9 +55,7 @@ class Dependency(Output):
def save(self): def save(self):
super().save() super().save()
if self.fs.version_aware: if self.fs.version_aware:
self.fs_path = self.fs.path.version_path( self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
self.fs_path, self.meta.version_id
)
def dumpd(self, **kwargs): def dumpd(self, **kwargs):
if self.fs.version_aware: if self.fs.version_aware:
+1 -3
View File
@@ -154,9 +154,7 @@ class ParamsDependency(Dependency):
try: try:
return load_path(self.fs_path, self.repo.fs) return load_path(self.fs_path, self.repo.fs)
except ParseError as exc: except ParseError as exc:
raise BadParamFileError( raise BadParamFileError(f"Unable to read parameters from '{self}'") from exc
f"Unable to read parameters from '{self}'"
) from exc
def get_hash(self): def get_hash(self):
info = self.read_params() info = self.read_params()
+7 -22
View File
@@ -35,9 +35,7 @@ class RepoDependency(Dependency):
} }
} }
def __init__( def __init__(self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs):
self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs
):
self.def_repo = def_repo self.def_repo = def_repo
self._objs: Dict[str, "HashFile"] = {} self._objs: Dict[str, "HashFile"] = {}
self._meta: Dict[str, "Meta"] = {} self._meta: Dict[str, "Meta"] = {}
@@ -100,17 +98,13 @@ class RepoDependency(Dependency):
# immutable, hence its impossible for checksum to change. # immutable, hence its impossible for checksum to change.
return False return False
def get_used_objs( def get_used_objs(self, **kwargs) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
self, **kwargs
) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
used, _, _ = self._get_used_and_obj(**kwargs) used, _, _ = self._get_used_and_obj(**kwargs)
return used return used
def _get_used_and_obj( def _get_used_and_obj(
self, obj_only: bool = False, **kwargs self, obj_only: bool = False, **kwargs
) -> Tuple[ ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"]:
Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"
]:
from dvc.config import NoRemoteError from dvc.config import NoRemoteError
from dvc.exceptions import NoOutputOrStageError, PathMissingError from dvc.exceptions import NoOutputOrStageError, PathMissingError
from dvc.utils import as_posix from dvc.utils import as_posix
@@ -163,9 +157,7 @@ class RepoDependency(Dependency):
used_obj_ids[object_store].update(oid for _, _, oid in obj) used_obj_ids[object_store].update(oid for _, _, oid in obj)
return used_obj_ids, meta, obj return used_obj_ids, meta, obj
def _check_circular_import( def _check_circular_import(self, odb: "ObjectDB", obj_ids: Set["HashInfo"]) -> None:
self, odb: "ObjectDB", obj_ids: Set["HashInfo"]
) -> None:
from dvc.exceptions import CircularImportError from dvc.exceptions import CircularImportError
from dvc.fs.dvc import DVCFileSystem from dvc.fs.dvc import DVCFileSystem
from dvc_data.hashfile.db.reference import ReferenceHashFileDB from dvc_data.hashfile.db.reference import ReferenceHashFileDB
@@ -187,10 +179,7 @@ class RepoDependency(Dependency):
for obj in iter_objs(): for obj in iter_objs():
if not isinstance(obj.fs, DVCFileSystem): if not isinstance(obj.fs, DVCFileSystem):
continue continue
if ( if obj.fs.repo_url in checked_urls or obj.fs.repo.root_dir in checked_urls:
obj.fs.repo_url in checked_urls
or obj.fs.repo.root_dir in checked_urls
):
continue continue
self_url = self.repo.url or self.repo.root_dir self_url = self.repo.url or self.repo.root_dir
if ( if (
@@ -221,9 +210,7 @@ class RepoDependency(Dependency):
) )
return meta return meta
def _make_repo( def _make_repo(self, locked: bool = True, **kwargs) -> "ContextManager[Repo]":
self, locked: bool = True, **kwargs
) -> "ContextManager[Repo]":
from dvc.external_repo import external_repo from dvc.external_repo import external_repo
d = self.def_repo d = self.def_repo
@@ -232,6 +219,4 @@ class RepoDependency(Dependency):
def _get_rev(self, locked: bool = True): def _get_rev(self, locked: bool = True):
d = self.def_repo d = self.def_repo
return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get( return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(self.PARAM_REV)
self.PARAM_REV
)
+8 -25
View File
@@ -63,9 +63,7 @@ def is_valid_filename(path):
def is_dvc_file(path): def is_dvc_file(path):
return os.path.isfile(path) and ( return os.path.isfile(path) and (is_valid_filename(path) or is_lock_file(path))
is_valid_filename(path) or is_lock_file(path)
)
def is_lock_file(path): def is_lock_file(path):
@@ -77,9 +75,7 @@ def is_git_ignored(repo, path):
from dvc.scm import NoSCMError from dvc.scm import NoSCMError
try: try:
return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored( return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(path)
path
)
except NoSCMError: except NoSCMError:
return False return False
@@ -152,9 +148,7 @@ class FileMixin:
# 4. when the file is git ignored # 4. when the file is git ignored
if not self.exists(): if not self.exists():
dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path) dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path)
raise StageFileDoesNotExistError( raise StageFileDoesNotExistError(self.path, dvc_ignored=dvc_ignored)
self.path, dvc_ignored=dvc_ignored
)
self._verify_filename() self._verify_filename()
if not self.repo.fs.isfile(self.path): if not self.repo.fs.isfile(self.path):
@@ -271,9 +265,7 @@ class ProjectFile(FileMixin):
@staticmethod @staticmethod
def _check_if_parametrized(stage, action: str = "dump") -> None: def _check_if_parametrized(stage, action: str = "dump") -> None:
if stage.raw_data.parametrized: if stage.raw_data.parametrized:
raise ParametrizedDumpError( raise ParametrizedDumpError(f"cannot {action} a parametrized {stage}")
f"cannot {action} a parametrized {stage}"
)
def _dump_pipeline_file(self, stage): def _dump_pipeline_file(self, stage):
self._check_if_parametrized(stage) self._check_if_parametrized(stage)
@@ -286,9 +278,7 @@ class ProjectFile(FileMixin):
data["stages"] = data.get("stages", {}) data["stages"] = data.get("stages", {})
existing_entry = stage.name in data["stages"] existing_entry = stage.name in data["stages"]
action = "Modifying" if existing_entry else "Adding" action = "Modifying" if existing_entry else "Adding"
logger.info( logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
"%s stage '%s' in '%s'", action, stage.name, self.relpath
)
if existing_entry: if existing_entry:
orig_stage_data = data["stages"][stage.name] orig_stage_data = data["stages"][stage.name]
@@ -300,9 +290,7 @@ class ProjectFile(FileMixin):
@property @property
def stage(self): def stage(self):
raise DvcException( raise DvcException("ProjectFile has multiple stages. Please specify it's name.")
"ProjectFile has multiple stages. Please specify it's name."
)
@cached_property @cached_property
def contents(self) -> Dict[str, Any]: def contents(self) -> Dict[str, Any]:
@@ -366,10 +354,7 @@ class ProjectFile(FileMixin):
def get_lockfile_schema(d): def get_lockfile_schema(d):
from dvc.schema import ( from dvc.schema import COMPILED_LOCKFILE_V1_SCHEMA, COMPILED_LOCKFILE_V2_SCHEMA
COMPILED_LOCKFILE_V1_SCHEMA,
COMPILED_LOCKFILE_V2_SCHEMA,
)
schema = { schema = {
LOCKFILE_VERSION.V1: COMPILED_LOCKFILE_V1_SCHEMA, LOCKFILE_VERSION.V1: COMPILED_LOCKFILE_V1_SCHEMA,
@@ -422,9 +407,7 @@ class Lockfile(FileMixin):
with modify_yaml(self.path, fs=self.repo.fs) as data: with modify_yaml(self.path, fs=self.repo.fs) as data:
version = LOCKFILE_VERSION.from_dict(data) version = LOCKFILE_VERSION.from_dict(data)
if version == LOCKFILE_VERSION.V1: if version == LOCKFILE_VERSION.V1:
logger.info( logger.info("Migrating lock file '%s' from v1 to v2", self.relpath)
"Migrating lock file '%s' from v1 to v2", self.relpath
)
migrate_lock_v1_to_v2(data, self.latest_version_info) migrate_lock_v1_to_v2(data, self.latest_version_info)
else: else:
if not data: if not data:
+9 -20
View File
@@ -150,8 +150,9 @@ class CyclicGraphError(DvcException):
class ConfirmRemoveError(DvcException): class ConfirmRemoveError(DvcException):
def __init__(self, path): def __init__(self, path):
super().__init__( super().__init__(
"unable to remove '{}' without a confirmation. Use " "unable to remove '{}' without a confirmation. Use `-f` to force.".format(
"`-f` to force.".format(path) path
)
) )
@@ -177,9 +178,7 @@ class BadMetricError(DvcException):
class RecursiveAddingWhileUsingFilename(DvcException): class RecursiveAddingWhileUsingFilename(DvcException):
def __init__(self): def __init__(self):
super().__init__( super().__init__("cannot use `fname` with multiple targets or `-R|--recursive`")
"cannot use `fname` with multiple targets or `-R|--recursive`"
)
class OverlappingOutputPathsError(DvcException): class OverlappingOutputPathsError(DvcException):
@@ -206,9 +205,7 @@ class FileMissingError(DvcException):
def __init__(self, path, hint=None): def __init__(self, path, hint=None):
self.path = path self.path = path
hint = "" if hint is None else f". {hint}" hint = "" if hint is None else f". {hint}"
super().__init__( super().__init__(f"Can't find '{path}' neither locally nor on remote{hint}")
f"Can't find '{path}' neither locally nor on remote{hint}"
)
class FileTransferError(DvcException): class FileTransferError(DvcException):
@@ -250,9 +247,7 @@ class CollectCacheError(DvcException):
class NoRemoteInExternalRepoError(DvcException): class NoRemoteInExternalRepoError(DvcException):
def __init__(self, url): def __init__(self, url):
super().__init__( super().__init__(f"No DVC remote is specified in target repository '{url}'.")
f"No DVC remote is specified in target repository '{url}'."
)
class NoOutputInExternalRepoError(DvcException): class NoOutputInExternalRepoError(DvcException):
@@ -277,8 +272,7 @@ class PathMissingError(DvcException):
" neither as a DVC output nor as a Git-tracked file." " neither as a DVC output nor as a Git-tracked file."
) )
default_msg_dvc_only = ( default_msg_dvc_only = (
"The path '{}' does not exist in the target repository '{}'" "The path '{}' does not exist in the target repository '{}' as an DVC output."
" as an DVC output."
) )
def __init__(self, path, repo, dvc_only=False): def __init__(self, path, repo, dvc_only=False):
@@ -294,7 +288,6 @@ class URLMissingError(DvcException):
class RemoteCacheRequiredError(DvcException): class RemoteCacheRequiredError(DvcException):
def __init__(self, scheme, fs_path): def __init__(self, scheme, fs_path):
super().__init__( super().__init__(
( (
"Current operation was unsuccessful because '{}' requires " "Current operation was unsuccessful because '{}' requires "
@@ -319,8 +312,7 @@ class NoOutputOrStageError(DvcException):
def __init__(self, target, file): def __init__(self, target, file):
super().__init__( super().__init__(
f"'{target}' " f"'{target}' does not exist as an output or a stage name in '{file}'"
f"does not exist as an output or a stage name in '{file}'"
) )
@@ -329,11 +321,8 @@ class MergeError(DvcException):
class CacheLinkError(DvcException): class CacheLinkError(DvcException):
SUPPORT_LINK = "See {} for more information.".format( SUPPORT_LINK = "See {} for more information.".format(
format_link( format_link("https://dvc.org/doc/user-guide/troubleshooting#cache-types")
"https://dvc.org/doc/user-guide/troubleshooting#cache-types"
)
) )
def __init__(self, fs_paths): def __init__(self, fs_paths):
+3 -9
View File
@@ -88,9 +88,7 @@ def external_repo(
raise NoRemoteInExternalRepoError(url) from exc raise NoRemoteInExternalRepoError(url) from exc
except OutputNotFoundError as exc: except OutputNotFoundError as exc:
if exc.repo is repo: if exc.repo is repo:
raise NoOutputInExternalRepoError( raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
exc.output, repo.root_dir, url
) from exc
raise raise
except FileMissingError as exc: except FileMissingError as exc:
raise PathMissingError(exc.path, url) from exc raise PathMissingError(exc.path, url) from exc
@@ -234,9 +232,7 @@ def _clone_default_branch(url, rev, for_write=False): # noqa: C901
os.path.join(clone_path, Git.GIT_DIR, "shallow") os.path.join(clone_path, Git.GIT_DIR, "shallow")
) )
if shallow: if shallow:
logger.debug( logger.debug("erepo: using shallow clone for branch '%s'", rev)
"erepo: using shallow clone for branch '%s'", rev
)
except CloneError: except CloneError:
git_dir = os.path.join(clone_path, ".git") git_dir = os.path.join(clone_path, ".git")
if os.path.exists(git_dir): if os.path.exists(git_dir):
@@ -292,8 +288,6 @@ def _remove(path):
try: try:
os_retry(remove)(path) os_retry(remove)(path)
except PermissionError: except PermissionError:
logger.warning( logger.warning("Failed to remove '%s'", relpath(path), exc_info=True)
"Failed to remove '%s'", relpath(path), exc_info=True
)
else: else:
remove(path) remove(path)
+1 -3
View File
@@ -72,9 +72,7 @@ def get_fs_config(config, **kwargs):
except KeyError: except KeyError:
from dvc.config import RemoteNotFoundError from dvc.config import RemoteNotFoundError
raise RemoteNotFoundError( # noqa: B904 raise RemoteNotFoundError(f"remote '{name}' doesn't exist") # noqa: B904
f"remote '{name}' doesn't exist"
)
else: else:
remote_conf = kwargs remote_conf = kwargs
return _resolve_remote_refs(config, remote_conf) return _resolve_remote_refs(config, remote_conf)
+1 -3
View File
@@ -58,9 +58,7 @@ class RichCallback(Callback):
@cached_property @cached_property
def task(self) -> "TaskID": def task(self) -> "TaskID":
return self.progress.add_task( return self.progress.add_task(**self._task_kwargs) # type: ignore[arg-type]
**self._task_kwargs # type: ignore[arg-type]
)
def __enter__(self): def __enter__(self):
return self return self
+5 -22
View File
@@ -6,16 +6,7 @@ import os
import posixpath import posixpath
import threading import threading
from contextlib import suppress from contextlib import suppress
from typing import ( from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
TYPE_CHECKING,
Any,
Callable,
Dict,
Optional,
Tuple,
Type,
Union,
)
from fsspec.spec import AbstractFileSystem from fsspec.spec import AbstractFileSystem
from funcy import wrap_with from funcy import wrap_with
@@ -126,9 +117,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
super().__init__() super().__init__()
if repo is None: if repo is None:
repo = self._make_repo( repo = self._make_repo(url=url, rev=rev, subrepos=subrepos, **repo_kwargs)
url=url, rev=rev, subrepos=subrepos, **repo_kwargs
)
assert repo is not None assert repo is not None
# pylint: disable=protected-access # pylint: disable=protected-access
repo_factory = repo._fs_conf["repo_factory"] repo_factory = repo._fs_conf["repo_factory"]
@@ -144,9 +133,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
relparts = () relparts = ()
assert repo is not None assert repo is not None
if repo.fs.path.isin(repo.fs.path.getcwd(), repo.root_dir): if repo.fs.path.isin(repo.fs.path.getcwd(), repo.root_dir):
relparts = repo.fs.path.relparts( relparts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.root_dir)
repo.fs.path.getcwd(), repo.root_dir
)
return self.root_marker + self.sep.join(relparts) return self.root_marker + self.sep.join(relparts)
self.path = Path(self.sep, getcwd=_getcwd) self.path = Path(self.sep, getcwd=_getcwd)
@@ -222,9 +209,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
scm=self.repo.scm, scm=self.repo.scm,
repo_factory=self.repo_factory, repo_factory=self.repo_factory,
) )
self._datafss[key] = DataFileSystem( self._datafss[key] = DataFileSystem(index=repo.index.data["repo"])
index=repo.index.data["repo"]
)
self._subrepos_trie[key] = repo self._subrepos_trie[key] = repo
def _is_dvc_repo(self, dir_path): def _is_dvc_repo(self, dir_path):
@@ -331,9 +316,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
ignore_subrepos = kwargs.get("ignore_subrepos", True) ignore_subrepos = kwargs.get("ignore_subrepos", True)
return self._info(key, path, ignore_subrepos=ignore_subrepos) return self._info(key, path, ignore_subrepos=ignore_subrepos)
def _info( # noqa: C901 def _info(self, key, path, ignore_subrepos=True, check_ignored=True): # noqa: C901
self, key, path, ignore_subrepos=True, check_ignored=True
):
repo, dvc_fs, subkey = self._get_subrepo_info(key) repo, dvc_fs, subkey = self._get_subrepo_info(key)
dvc_info = None dvc_info = None
+10 -26
View File
@@ -30,9 +30,7 @@ class DvcIgnorePatterns(DvcIgnore):
from pathspec.patterns.gitwildmatch import _DIR_MARK from pathspec.patterns.gitwildmatch import _DIR_MARK
if pattern_list and isinstance(pattern_list[0], str): if pattern_list and isinstance(pattern_list[0], str):
pattern_list = [ pattern_list = [PatternInfo(pattern, "") for pattern in pattern_list]
PatternInfo(pattern, "") for pattern in pattern_list
]
self.sep = sep self.sep = sep
self.pattern_list = pattern_list self.pattern_list = pattern_list
@@ -40,20 +38,14 @@ class DvcIgnorePatterns(DvcIgnore):
self.regex_pattern_list = [] self.regex_pattern_list = []
for count, pattern in enumerate(pattern_list): for count, pattern in enumerate(pattern_list):
pattern, group = GitWildMatchPattern.pattern_to_regex( pattern, group = GitWildMatchPattern.pattern_to_regex(pattern.patterns)
pattern.patterns
)
if pattern: if pattern:
pattern = pattern.replace( pattern = pattern.replace(f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>")
f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>"
)
self.regex_pattern_list.append((pattern, group)) self.regex_pattern_list.append((pattern, group))
self.ignore_spec = [ self.ignore_spec = [
(ignore, re.compile("|".join(item[0] for item in group))) (ignore, re.compile("|".join(item[0] for item in group)))
for ignore, group in groupby( for ignore, group in groupby(self.regex_pattern_list, lambda x: x[1])
self.regex_pattern_list, lambda x: x[1]
)
if ignore is not None if ignore is not None
] ]
@@ -64,9 +56,7 @@ class DvcIgnorePatterns(DvcIgnore):
with fs.open(path, encoding="utf-8") as fobj: with fs.open(path, encoding="utf-8") as fobj:
path_spec_lines = [ path_spec_lines = [
PatternInfo(line, f"{name}:{line_no + 1}:{line}") PatternInfo(line, f"{name}:{line_no + 1}:{line}")
for line_no, line in enumerate( for line_no, line in enumerate(map(str.strip, fobj.readlines()))
map(str.strip, fobj.readlines())
)
if line and not (line.strip().startswith("#")) if line and not (line.strip().startswith("#"))
] ]
@@ -158,9 +148,7 @@ class DvcIgnorePatterns(DvcIgnore):
return bool(self.pattern_list) return bool(self.pattern_list)
CheckIgnoreResult = namedtuple( CheckIgnoreResult = namedtuple("CheckIgnoreResult", ["file", "match", "patterns"])
"CheckIgnoreResult", ["file", "match", "patterns"]
)
def _no_match(path): def _no_match(path):
@@ -249,9 +237,7 @@ class DvcIgnoreFilter:
dnames = [] dnames = []
for dname in dnames: for dname in dnames:
self._update_sub_repo( self._update_sub_repo(self.fs.path.join(dirname, dname), ignore_trie)
self.fs.path.join(dirname, dname), ignore_trie
)
def _update_sub_repo(self, path, ignore_trie: Trie): def _update_sub_repo(self, path, ignore_trie: Trie):
from dvc.repo import Repo from dvc.repo import Repo
@@ -409,9 +395,7 @@ class DvcIgnoreFilter:
# https://github.com/iterative/dvc/issues/5046 # https://github.com/iterative/dvc/issues/5046
full_target = self.fs.path.abspath(target) full_target = self.fs.path.abspath(target)
if not self._outside_repo(full_target): if not self._outside_repo(full_target):
dirname, basename = self.fs.path.split( dirname, basename = self.fs.path.split(self.fs.path.normpath(full_target))
self.fs.path.normpath(full_target)
)
pattern = self._get_trie_pattern(dirname) pattern = self._get_trie_pattern(dirname)
if pattern: if pattern:
matches = pattern.matches( matches = pattern.matches(
@@ -433,9 +417,9 @@ class DvcIgnoreFilter:
return self.is_ignored_file(path, ignore_subrepos) return self.is_ignored_file(path, ignore_subrepos)
if fs.isdir(path): if fs.isdir(path):
return self.is_ignored_dir(path, ignore_subrepos) return self.is_ignored_dir(path, ignore_subrepos)
return self.is_ignored_file( return self.is_ignored_file(path, ignore_subrepos) or self.is_ignored_dir(
path, ignore_subrepos path, ignore_subrepos
) or self.is_ignored_dir(path, ignore_subrepos) )
def init(path): def init(path):
+2 -4
View File
@@ -30,8 +30,7 @@ def get_dvc_info():
info = [ info = [
dvc_version, dvc_version,
"-" * len(dvc_version), "-" * len(dvc_version),
f"Platform: Python {platform.python_version()} on " f"Platform: Python {platform.python_version()} on {platform.platform()}",
f"{platform.platform()}",
f"Subprojects:{_get_subprojects()}", f"Subprojects:{_get_subprojects()}",
f"Supports:{_get_supported_remotes()}", f"Supports:{_get_supported_remotes()}",
] ]
@@ -125,8 +124,7 @@ def _get_supported_remotes():
dependencies = [] dependencies = []
for requirement in fs_cls.REQUIRES: for requirement in fs_cls.REQUIRES:
dependencies.append( dependencies.append(
f"{requirement} = " f"{requirement} = {importlib_metadata.version(requirement)}"
f"{importlib_metadata.version(requirement)}"
) )
remote_info = scheme remote_info = scheme
+4 -10
View File
@@ -20,9 +20,7 @@ FAILED_TO_LOCK_MESSAGE = (
"Unable to acquire lock. Most likely another DVC process is running or " "Unable to acquire lock. Most likely another DVC process is running or "
"was terminated abruptly. Check the page {} for other possible reasons " "was terminated abruptly. Check the page {} for other possible reasons "
"and to learn how to resolve this." "and to learn how to resolve this."
).format( ).format(format_link("https://dvc.org/doc/user-guide/troubleshooting#lock-issue"))
format_link("https://dvc.org/doc/user-guide/troubleshooting#lock-issue")
)
class LockError(DvcException): class LockError(DvcException):
@@ -61,9 +59,7 @@ class LockBase(ABC):
class LockNoop(LockBase): class LockNoop(LockBase):
def __init__( def __init__(self, *args, **kwargs): # pylint: disable=super-init-not-called
self, *args, **kwargs
): # pylint: disable=super-init-not-called
self._lock = False self._lock = False
def lock(self): def lock(self):
@@ -107,10 +103,8 @@ class Lock(LockBase):
with Tqdm( with Tqdm(
bar_format="{desc}", bar_format="{desc}",
disable=not self._friendly, disable=not self._friendly,
desc=( desc="If DVC froze, see `hardlink_lock` in {}".format(
"If DVC froze, see `hardlink_lock` in {}".format( format_link("https://man.dvc.org/config#core")
format_link("https://man.dvc.org/config#core")
)
), ),
): ):
self._lock = zc.lockfile.LockFile(self._lockfile) self._lock = zc.lockfile.LockFile(self._lockfile)
+7 -16
View File
@@ -159,9 +159,7 @@ class LoggerHandler(logging.StreamHandler):
pass # noqa: S110 pass # noqa: S110
msg = self.format(record) msg = self.format(record)
Tqdm.write( Tqdm.write(msg, file=self.stream, end=getattr(self, "terminator", "\n"))
msg, file=self.stream, end=getattr(self, "terminator", "\n")
)
self.flush() self.flush()
except (BrokenPipeError, RecursionError): except (BrokenPipeError, RecursionError):
raise raise
@@ -171,9 +169,7 @@ class LoggerHandler(logging.StreamHandler):
def _is_verbose(): def _is_verbose():
return ( return (
logging.NOTSET logging.NOTSET < logging.getLogger("dvc").getEffectiveLevel() <= logging.DEBUG
< logging.getLogger("dvc").getEffectiveLevel()
<= logging.DEBUG
) )
@@ -186,16 +182,11 @@ def _iter_causes(exc):
def _stack_trace(exc_info): def _stack_trace(exc_info):
import traceback import traceback
return ( return "\n{red}{line}{nc}\n{trace}{red}{line}{nc}".format(
"\n" red=colorama.Fore.RED,
"{red}{line}{nc}\n" line="-" * 60,
"{trace}" trace="".join(traceback.format_exception(*exc_info)),
"{red}{line}{nc}".format( nc=colorama.Fore.RESET,
red=colorama.Fore.RED,
line="-" * 60,
trace="".join(traceback.format_exception(*exc_info)),
nc=colorama.Fore.RESET,
)
) )
+1 -3
View File
@@ -67,9 +67,7 @@ class MachineBackends(Mapping):
initialized = self.initialized.get(key) initialized = self.initialized.get(key)
if not initialized: if not initialized:
backend = self.backends[key] backend = self.backends[key]
initialized = backend( initialized = backend(os.path.join(self.tmp_dir, key), **self.kwargs)
os.path.join(self.tmp_dir, key), **self.kwargs
)
self.initialized[key] = initialized self.initialized[key] = initialized
return initialized return initialized
+1 -3
View File
@@ -21,9 +21,7 @@ class BaseMachineBackend(ABC):
"""Stop and destroy all instances of the specified machine.""" """Stop and destroy all instances of the specified machine."""
@abstractmethod @abstractmethod
def instances( def instances(self, name: Optional[str] = None, **config) -> Iterator[dict]:
self, name: Optional[str] = None, **config
) -> Iterator[dict]:
"""Iterate over status of all instances of the specified machine.""" """Iterate over status of all instances of the specified machine."""
def close(self): # noqa: B027 def close(self): # noqa: B027
+2 -6
View File
@@ -56,12 +56,8 @@ class TerraformBackend(BaseMachineBackend):
create = partialmethod(_tpi_func, "create") # type: ignore[assignment] create = partialmethod(_tpi_func, "create") # type: ignore[assignment]
destroy = partialmethod(_tpi_func, "destroy") # type: ignore[assignment] destroy = partialmethod(_tpi_func, "destroy") # type: ignore[assignment]
instances = partialmethod( instances = partialmethod(_tpi_func, "instances") # type: ignore[assignment]
_tpi_func, "instances" run_shell = partialmethod(_tpi_func, "run_shell") # type: ignore[assignment]
) # type: ignore[assignment]
run_shell = partialmethod(
_tpi_func, "run_shell"
) # type: ignore[assignment]
def get_executor_kwargs(self, name: str, **config) -> dict: def get_executor_kwargs(self, name: str, **config) -> dict:
with self.make_tpi(name) as tpi: with self.make_tpi(name) as tpi:
+13 -38
View File
@@ -151,9 +151,7 @@ def _merge_data(s_list):
d[key].update({}) d[key].update({})
continue continue
if not isinstance(key, dict): if not isinstance(key, dict):
raise ValueError( # noqa: TRY004 raise ValueError(f"'{type(key).__name__}' not supported.") # noqa: TRY004
f"'{type(key).__name__}' not supported."
)
for k, flags in key.items(): for k, flags in key.items():
if not isinstance(flags, dict): if not isinstance(flags, dict):
@@ -336,9 +334,7 @@ class Output:
if fs_config is not None: if fs_config is not None:
fs_kwargs.update(**fs_config) fs_kwargs.update(**fs_config)
fs_cls, fs_config, fs_path = get_cloud_fs( fs_cls, fs_config, fs_path = get_cloud_fs(self.repo, url=path, **fs_kwargs)
self.repo, url=path, **fs_kwargs
)
self.fs = fs_cls(**fs_config) self.fs = fs_cls(**fs_config)
if ( if (
@@ -500,9 +496,7 @@ class Output:
@property @property
def cache_path(self): def cache_path(self):
return self.odb.fs.unstrip_protocol( return self.odb.fs.unstrip_protocol(self.odb.oid_to_path(self.hash_info.value))
self.odb.oid_to_path(self.hash_info.value)
)
def get_hash(self): def get_hash(self):
_, hash_info = self._get_hash_meta() _, hash_info = self._get_hash_meta()
@@ -691,9 +685,7 @@ class Output:
dry_run=True, dry_run=True,
) )
if not self.IS_DEPENDENCY: if not self.IS_DEPENDENCY:
logger.debug( logger.debug("Output '%s' doesn't use cache. Skipping saving.", self)
"Output '%s' doesn't use cache. Skipping saving.", self
)
self.hash_info = self.obj.hash_info self.hash_info = self.obj.hash_info
self.files = None self.files = None
@@ -724,9 +716,7 @@ class Output:
if self.use_cache: if self.use_cache:
granular = ( granular = (
self.is_dir_checksum self.is_dir_checksum and filter_info and filter_info != self.fs_path
and filter_info
and filter_info != self.fs_path
) )
if granular: if granular:
obj = self._commit_granular_dir(filter_info) obj = self._commit_granular_dir(filter_info)
@@ -757,9 +747,7 @@ class Output:
self.set_exec() self.set_exec()
def _commit_granular_dir(self, filter_info) -> Optional["HashFile"]: def _commit_granular_dir(self, filter_info) -> Optional["HashFile"]:
prefix = self.fs.path.parts( prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
self.fs.path.relpath(filter_info, self.fs_path)
)
staging, _, save_obj = build( staging, _, save_obj = build(
self.odb, self.odb,
self.fs_path, self.fs_path,
@@ -794,9 +782,7 @@ class Output:
ret.update(split_file_meta_from_cloud(meta_d)) ret.update(split_file_meta_from_cloud(meta_d))
if self.is_in_repo: if self.is_in_repo:
path = self.fs.path.as_posix( path = self.fs.path.as_posix(relpath(self.fs_path, self.stage.wdir))
relpath(self.fs_path, self.stage.wdir)
)
else: else:
path = self.def_path path = self.def_path
@@ -837,16 +823,13 @@ class Output:
if obj: if obj:
obj = cast("Tree", obj) obj = cast("Tree", obj)
ret[self.PARAM_FILES] = [ ret[self.PARAM_FILES] = [
split_file_meta_from_cloud(f) split_file_meta_from_cloud(f) for f in obj.as_list(with_meta=True)
for f in obj.as_list(with_meta=True)
] ]
return ret return ret
def verify_metric(self): def verify_metric(self):
if self.fs.protocol != "local": if self.fs.protocol != "local":
raise DvcException( raise DvcException(f"verify metric is not supported for {self.protocol}")
f"verify metric is not supported for {self.protocol}"
)
if not self.metric: if not self.metric:
return return
@@ -902,9 +885,7 @@ class Output:
) -> Optional[Tuple[bool, Optional[bool]]]: ) -> Optional[Tuple[bool, Optional[bool]]]:
if not self.use_cache: if not self.use_cache:
if progress_callback != DEFAULT_CALLBACK: if progress_callback != DEFAULT_CALLBACK:
progress_callback.relative_update( progress_callback.relative_update(self.get_files_number(filter_info))
self.get_files_number(filter_info)
)
return None return None
obj = self.get_obj(filter_info=filter_info) obj = self.get_obj(filter_info=filter_info)
@@ -1071,9 +1052,7 @@ class Output:
obj = self.get_obj() obj = self.get_obj()
if filter_info and filter_info != self.fs_path: if filter_info and filter_info != self.fs_path:
assert obj assert obj
prefix = self.fs.path.parts( prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
self.fs.path.relpath(filter_info, self.fs_path)
)
obj = cast(Tree, obj) obj = cast(Tree, obj)
return obj.filter(prefix) return obj.filter(prefix)
return cast(Tree, obj) return cast(Tree, obj)
@@ -1181,14 +1160,10 @@ class Output:
other.pop(opt, None) other.pop(opt, None)
if my != other: if my != other:
raise MergeError( raise MergeError("unable to auto-merge outputs with different options")
"unable to auto-merge outputs with different options"
)
if not out.is_dir_checksum: if not out.is_dir_checksum:
raise MergeError( raise MergeError("unable to auto-merge outputs that are not directories")
"unable to auto-merge outputs that are not directories"
)
def merge(self, ancestor, other, allowed=None): def merge(self, ancestor, other, allowed=None):
from dvc_data.hashfile.tree import MergeError as TreeMergeError from dvc_data.hashfile.tree import MergeError as TreeMergeError
+12 -19
View File
@@ -72,9 +72,7 @@ def _format_preamble(msg: str, path: str, spacing: str = " ") -> str:
def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn": def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn":
spacing = ( spacing = (
"\n" "\n" if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded)) else " "
if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded))
else " "
) )
message = _format_preamble(msg, path, spacing) + str(exc) message = _format_preamble(msg, path, spacing) + str(exc)
@@ -116,8 +114,7 @@ def check_interpolations(data: "DictStrAny", where: str, path: str):
def func(s: "DictStrAny") -> None: def func(s: "DictStrAny") -> None:
if is_interpolated_string(s): if is_interpolated_string(s):
raise ResolveError( raise ResolveError(
_format_preamble(f"'{where}'", path) _format_preamble(f"'{where}'", path) + "interpolating is not allowed"
+ "interpolating is not allowed"
) )
return recurse(func)(data) return recurse(func)(data)
@@ -188,9 +185,7 @@ class DataResolver:
def resolve(self): def resolve(self):
"""Used for testing purposes, otherwise use resolve_one().""" """Used for testing purposes, otherwise use resolve_one()."""
data = join(map(self.resolve_one, self.get_keys())) data = join(map(self.resolve_one, self.get_keys()))
logger.trace( # type: ignore[attr-defined] logger.trace("Resolved dvc.yaml:\n%s", data) # type: ignore[attr-defined]
"Resolved dvc.yaml:\n%s", data
)
return {STAGES_KWD: data} return {STAGES_KWD: data}
def has_key(self, key: str): def has_key(self, key: str):
@@ -203,9 +198,9 @@ class DataResolver:
return False return False
if key: if key:
return isinstance( return isinstance(definition, ForeachDefinition) and definition.has_member(
definition, ForeachDefinition key
) and definition.has_member(key) )
return not isinstance(definition, ForeachDefinition) return not isinstance(definition, ForeachDefinition)
@collecting @collecting
@@ -309,9 +304,7 @@ class EntryDefinition:
value, skip_interpolation_checks=skip_checks, key=key value, skip_interpolation_checks=skip_checks, key=key
) )
except (ParseError, KeyNotInContext) as exc: except (ParseError, KeyNotInContext) as exc:
format_and_raise( format_and_raise(exc, f"'{self.where}.{self.name}.{key}'", self.relpath)
exc, f"'{self.where}.{self.name}.{key}'", self.relpath
)
class IterationPair(NamedTuple): class IterationPair(NamedTuple):
@@ -354,9 +347,7 @@ class ForeachDefinition:
try: try:
iterable = self.context.resolve(self.foreach_data, unwrap=False) iterable = self.context.resolve(self.foreach_data, unwrap=False)
except (ContextError, ParseError) as exc: except (ContextError, ParseError) as exc:
format_and_raise( format_and_raise(exc, f"'{self.where}.{self.name}.foreach'", self.relpath)
exc, f"'{self.where}.{self.name}.foreach'", self.relpath
)
# foreach data can be a resolved dictionary/list. # foreach data can be a resolved dictionary/list.
self._check_is_map_or_seq(iterable) self._check_is_map_or_seq(iterable)
@@ -381,8 +372,10 @@ class ForeachDefinition:
if warn_for: if warn_for:
linking_verb = "is" if len(warn_for) == 1 else "are" linking_verb = "is" if len(warn_for) == 1 else "are"
logger.warning( logger.warning(
"%s %s already specified, " (
"will be overwritten for stages generated from '%s'", "%s %s already specified, "
"will be overwritten for stages generated from '%s'"
),
" and ".join(warn_for), " and ".join(warn_for),
linking_verb, linking_verb,
self.name, self.name,
+7 -22
View File
@@ -48,9 +48,7 @@ class MergeError(ContextError):
self.key = key self.key = key
to_node = into[key] to_node = into[key]
if not isinstance(to_node, Node) or not isinstance(new, Node): if not isinstance(to_node, Node) or not isinstance(new, Node):
super().__init__( super().__init__(f"cannot merge '{key}' as it already exists in {into}")
f"cannot merge '{key}' as it already exists in {into}"
)
return return
assert isinstance(to_node, Node) assert isinstance(to_node, Node)
@@ -138,9 +136,7 @@ class Node:
@dataclass @dataclass
class Value(Node): class Value(Node):
_value: Any _value: Any
meta: Meta = field( meta: Meta = field(compare=False, default_factory=_default_meta, repr=False)
compare=False, default_factory=_default_meta, repr=False
)
def __repr__(self): def __repr__(self):
return repr(self._value) return repr(self._value)
@@ -182,10 +178,7 @@ class Container(Node, ABC): # noqa: B024
assert meta assert meta
container = CtxDict if isinstance(value, dict) else CtxList container = CtxDict if isinstance(value, dict) else CtxList
return container(value, meta=meta) return container(value, meta=meta)
msg = ( msg = f"Unsupported value of type '{type(value).__name__}' in '{meta}'"
"Unsupported value of type "
f"'{type(value).__name__}' in '{meta}'"
)
raise TypeError(msg) raise TypeError(msg)
def __repr__(self): def __repr__(self):
@@ -219,9 +212,7 @@ class Container(Node, ABC): # noqa: B024
try: try:
d = self[index] d = self[index]
except LookupError as exc: except LookupError as exc:
raise ValueError( raise ValueError(f"Could not find '{index}' in {self.data}") from exc
f"Could not find '{index}' in {self.data}"
) from exc
if not rems: if not rems:
return d return d
@@ -380,9 +371,7 @@ class Context(CtxDict):
data = {key: data[key] for key in select_keys} data = {key: data[key] for key in select_keys}
except KeyError as exc: except KeyError as exc:
key, *_ = exc.args key, *_ = exc.args
raise ParamsLoadError( raise ParamsLoadError(f"could not find '{key}' in '{path}'") from exc
f"could not find '{key}' in '{path}'"
) from exc
meta = Meta(source=path, local=False) meta = Meta(source=path, local=False)
ctx = cls(data, meta=meta) ctx = cls(data, meta=meta)
@@ -480,9 +469,7 @@ class Context(CtxDict):
But for now, just `item` and `key`, this should do. But for now, just `item` and `key`, this should do.
""" """
# using dict to make the error messages ordered # using dict to make the error messages ordered
new = dict.fromkeys( new = dict.fromkeys([key for key in keys if key not in self._reserved_keys])
[key for key in keys if key not in self._reserved_keys]
)
self._reserved_keys.update(new) self._reserved_keys.update(new)
try: try:
yield yield
@@ -542,9 +529,7 @@ class Context(CtxDict):
if is_exact_string(src, matches): if is_exact_string(src, matches):
# replace "${enabled}", if `enabled` is a boolean, with it's actual # replace "${enabled}", if `enabled` is a boolean, with it's actual
# value rather than it's string counterparts. # value rather than it's string counterparts.
expr = get_expression( expr = get_expression(matches[0], skip_checks=skip_interpolation_checks)
matches[0], skip_checks=skip_interpolation_checks
)
value = self.select(expr, unwrap=unwrap) value = self.select(expr, unwrap=unwrap)
validate_value(value, key) validate_value(value, key)
return value return value
+2 -7
View File
@@ -99,7 +99,6 @@ def _(obj: dict): # noqa: C901
result = "" result = ""
for k, v in flatten(obj).items(): for k, v in flatten(obj).items():
if isinstance(v, bool): if isinstance(v, bool):
if v: if v:
result += f"--{k} " result += f"--{k} "
@@ -115,9 +114,7 @@ def _(obj: dict): # noqa: C901
if isinstance(i, str): if isinstance(i, str):
i = escape_str(i) i = escape_str(i)
elif isinstance(i, Iterable): elif isinstance(i, Iterable):
raise ParseError( raise ParseError(f"Cannot interpolate nested iterable in '{k}'")
f"Cannot interpolate nested iterable in '{k}'"
)
if config.get("list", "nargs") == "append": if config.get("list", "nargs") == "append":
result += f"--{k} {i} " result += f"--{k} {i} "
@@ -206,9 +203,7 @@ def validate_value(value, key):
if not_primitive and not_foreach: if not_primitive and not_foreach:
if isinstance(value, dict) and key == "cmd": if isinstance(value, dict) and key == "cmd":
return True return True
raise ParseError( raise ParseError(f"Cannot interpolate data of type '{type(value).__name__}'")
f"Cannot interpolate data of type '{type(value).__name__}'"
)
def str_interpolate( def str_interpolate(
+1 -3
View File
@@ -9,9 +9,7 @@ META_KWD = "meta"
def lockfile_version_schema(value): def lockfile_version_schema(value):
expected = [LOCKFILE_VERSION.V2.value] # pylint: disable=no-member expected = [LOCKFILE_VERSION.V2.value] # pylint: disable=no-member
msg = "invalid schema version {}, expected one of {}".format( msg = f"invalid schema version {value}, expected one of {expected}"
value, expected
)
return validators.Any(*expected, msg=msg)(value) return validators.Any(*expected, msg=msg)(value)
+1 -3
View File
@@ -13,9 +13,7 @@ class ProcessNotFoundError(DvcException):
class TimeoutExpired(DvcException): class TimeoutExpired(DvcException):
def __init__(self, cmd, timeout): def __init__(self, cmd, timeout):
super().__init__( super().__init__(f"'{cmd}' did not complete before timeout '{timeout}'")
f"'{cmd}' did not complete before timeout '{timeout}'"
)
self.cmd = cmd self.cmd = cmd
self.timeout = timeout self.timeout = timeout
+1 -3
View File
@@ -91,9 +91,7 @@ class ProcessManager:
raise UnsupportedSignalError(sig) raise UnsupportedSignalError(sig)
def handle_closed_process(): def handle_closed_process():
logging.warning( logging.warning("Process %s had already aborted unexpectedly.", name)
"Process %s had already aborted unexpectedly.", name
)
process_info.returncode = -1 process_info.returncode = -1
self[name] = process_info self[name] = process_info
+3 -11
View File
@@ -28,9 +28,7 @@ class Tqdm(tqdm):
"{postfix[info]}{n_fmt}/{total_fmt}" "{postfix[info]}{n_fmt}/{total_fmt}"
" [{elapsed}<{remaining}, {rate_fmt:>11}]" " [{elapsed}<{remaining}, {rate_fmt:>11}]"
) )
BAR_FMT_NOTOTAL = ( BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
"{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
)
BYTES_DEFAULTS = { BYTES_DEFAULTS = {
"unit": "B", "unit": "B",
"unit_scale": True, "unit_scale": True,
@@ -75,11 +73,7 @@ class Tqdm(tqdm):
if not disable: if not disable:
disable = logger.getEffectiveLevel() > level disable = logger.getEffectiveLevel() > level
# auto-disable based on TTY # auto-disable based on TTY
if ( if not disable and not env2bool(DVC_IGNORE_ISATTY) and hasattr(file, "isatty"):
not disable
and not env2bool(DVC_IGNORE_ISATTY)
and hasattr(file, "isatty")
):
disable = not file.isatty() disable = not file.isatty()
super().__init__( super().__init__(
iterable=iterable, iterable=iterable,
@@ -95,9 +89,7 @@ class Tqdm(tqdm):
if bar_format is None: if bar_format is None:
if self.__len__(): if self.__len__():
self.bar_format = ( self.bar_format = (
self.BAR_FMT_DEFAULT_NESTED self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT
if self.pos
else self.BAR_FMT_DEFAULT
) )
else: else:
self.bar_format = self.BAR_FMT_NOTOTAL self.bar_format = self.BAR_FMT_NOTOTAL
+1 -3
View File
@@ -24,9 +24,7 @@ def ask(prompt: str, limited_to: Optional[Collection[str]] = None):
if answer in limited_to: if answer in limited_to:
return answer return answer
logger.info( logger.info("Your response must be one of: %s. Please try again.", limited_to)
"Your response must be one of: %s. Please try again.", limited_to
)
def confirm(statement: str) -> bool: def confirm(statement: str) -> bool:
+1 -7
View File
@@ -2,13 +2,7 @@ import json
from collections import defaultdict from collections import defaultdict
from typing import Dict, List, Union from typing import Dict, List, Union
from dvc.render import ( from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD
REVISION_FIELD,
REVISIONS_KEY,
SRC_FIELD,
TYPE_KEY,
VERSION_FIELD,
)
from dvc.render.converter.image import ImageConverter from dvc.render.converter.image import ImageConverter
from dvc.render.converter.vega import VegaConverter from dvc.render.converter.vega import VegaConverter
+5 -9
View File
@@ -81,9 +81,9 @@ def _is_datapoints(lst: List[Dict]):
to unexpected behavior to unexpected behavior
""" """
return all(isinstance(item, dict) for item in lst) and set( return all(isinstance(item, dict) for item in lst) and set(first(lst).keys()) == {
first(lst).keys() key for keys in lst for key in keys
) == {key for keys in lst for key in keys} }
def get_datapoints(file_content: Dict): def get_datapoints(file_content: Dict):
@@ -195,7 +195,6 @@ class VegaConverter(Converter):
return "x" return "x"
def flat_datapoints(self, revision): # noqa: C901 def flat_datapoints(self, revision): # noqa: C901
file2datapoints, properties = self.convert() file2datapoints, properties = self.convert()
props_update = {} props_update = {}
@@ -218,7 +217,7 @@ class VegaConverter(Converter):
num_ys = len(ys) num_ys = len(ys)
if num_xs > 1 and num_xs != num_ys: if num_xs > 1 and num_xs != num_ys:
raise DvcException( raise DvcException(
f"Cannot have different number of x and y data sources. Found " "Cannot have different number of x and y data sources. Found "
f"{num_xs} x and {num_ys} y data sources." f"{num_xs} x and {num_ys} y data sources."
) )
@@ -316,9 +315,7 @@ def _update_from_field(
source_field = field source_field = field
if len(source_datapoints) != len(target_datapoints): if len(source_datapoints) != len(target_datapoints):
raise IndexError( raise IndexError("Source and target datapoints must have the same length")
"Source and target datapoints must have the same length"
)
for index, datapoint in enumerate(target_datapoints): for index, datapoint in enumerate(target_datapoints):
source_datapoint = source_datapoints[index] source_datapoint = source_datapoints[index]
@@ -332,6 +329,5 @@ def _update_from_index(datapoints: List[Dict], new_field: str):
def _update_all(datapoints: List[Dict], update_dict: Dict): def _update_all(datapoints: List[Dict], update_dict: Dict):
for datapoint in datapoints: for datapoint in datapoints:
datapoint.update(update_dict) datapoint.update(update_dict)
+3 -10
View File
@@ -62,7 +62,6 @@ def match_defs_renderers(
out=None, out=None,
templates_dir: Optional["StrPath"] = None, templates_dir: Optional["StrPath"] = None,
): ):
from dvc_render import ImageRenderer, VegaRenderer from dvc_render import ImageRenderer, VegaRenderer
plots_data = PlotsData(data) plots_data = PlotsData(data)
@@ -80,9 +79,7 @@ def match_defs_renderers(
for rev, inner_id, plot_definition in group: for rev, inner_id, plot_definition in group:
plot_sources = infer_data_sources(inner_id, plot_definition) plot_sources = infer_data_sources(inner_id, plot_definition)
definitions_data = plots_data.get_definition_data( definitions_data = plots_data.get_definition_data(plot_sources, rev)
plot_sources, rev
)
if ImageRenderer.matches(inner_id, None): if ImageRenderer.matches(inner_id, None):
renderer_cls = ImageRenderer renderer_cls = ImageRenderer
@@ -91,9 +88,7 @@ def match_defs_renderers(
renderer_cls = VegaRenderer renderer_cls = VegaRenderer
renderer_id = plot_id renderer_id = plot_id
converter = _get_converter( converter = _get_converter(renderer_cls, inner_id, props, definitions_data)
renderer_cls, inner_id, props, definitions_data
)
dps, rev_props = converter.flat_datapoints(rev) dps, rev_props = converter.flat_datapoints(rev)
if not final_props and rev_props: if not final_props and rev_props:
@@ -103,7 +98,5 @@ def match_defs_renderers(
if "title" not in final_props: if "title" not in final_props:
final_props["title"] = renderer_id final_props["title"] = renderer_id
if renderer_cls is not None: if renderer_cls is not None:
renderers.append( renderers.append(renderer_cls(plot_datapoints, renderer_id, **final_props))
renderer_cls(plot_datapoints, renderer_id, **final_props)
)
return renderers return renderers
+7 -27
View File
@@ -3,16 +3,7 @@ import os
from collections import defaultdict from collections import defaultdict
from contextlib import contextmanager from contextlib import contextmanager
from functools import wraps from functools import wraps
from typing import ( from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple, Union, cast
TYPE_CHECKING,
Callable,
Iterable,
List,
Optional,
Tuple,
Union,
cast,
)
from dvc.exceptions import FileMissingError from dvc.exceptions import FileMissingError
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
@@ -148,9 +139,7 @@ class Repo:
import hashlib import hashlib
root_dir_hash = hashlib.sha224( root_dir_hash = hashlib.sha224(self.root_dir.encode("utf-8")).hexdigest()
self.root_dir.encode("utf-8")
).hexdigest()
db_dir = self.fs.path.join( db_dir = self.fs.path.join(
base_db_dir, base_db_dir,
@@ -230,9 +219,7 @@ class Repo:
self.lock = make_lock( self.lock = make_lock(
self.fs.path.join(self.tmp_dir, "lock"), self.fs.path.join(self.tmp_dir, "lock"),
tmp_dir=self.tmp_dir, tmp_dir=self.tmp_dir,
hardlink_lock=self.config["core"].get( hardlink_lock=self.config["core"].get("hardlink_lock", False),
"hardlink_lock", False
),
friendly=True, friendly=True,
) )
state_db_dir = self._get_database_dir("state") state_db_dir = self._get_database_dir("state")
@@ -338,8 +325,7 @@ class Repo:
from dvc.machine import MachineManager from dvc.machine import MachineManager
if self.tmp_dir and ( if self.tmp_dir and (
self.config["feature"].get("machine", False) self.config["feature"].get("machine", False) or env2bool("DVC_TEST")
or env2bool("DVC_TEST")
): ):
return MachineManager(self) return MachineManager(self)
return None return None
@@ -393,14 +379,10 @@ class Repo:
return os.path.join(root_dir, cls.DVC_DIR) return os.path.join(root_dir, cls.DVC_DIR)
@staticmethod @staticmethod
def init( def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo":
root_dir=os.curdir, no_scm=False, force=False, subdir=False
) -> "Repo":
from dvc.repo.init import init from dvc.repo.init import init
return init( return init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir)
root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir
)
def unprotect(self, target): def unprotect(self, target):
return self.odb.repo.unprotect(target) return self.odb.repo.unprotect(target)
@@ -530,9 +512,7 @@ class Repo:
def dvcfs(self) -> "DVCFileSystem": def dvcfs(self) -> "DVCFileSystem":
from dvc.fs.dvc import DVCFileSystem from dvc.fs.dvc import DVCFileSystem
return DVCFileSystem( return DVCFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf)
repo=self, subrepos=self.subrepos, **self._fs_conf
)
@cached_property @cached_property
def index_db_dir(self): def index_db_dir(self):
+2 -9
View File
@@ -15,12 +15,7 @@ from dvc.exceptions import (
) )
from dvc.repo.scm_context import scm_context from dvc.repo.scm_context import scm_context
from dvc.ui import ui from dvc.ui import ui
from dvc.utils import ( from dvc.utils import LARGE_DIR_SIZE, glob_targets, resolve_output, resolve_paths
LARGE_DIR_SIZE,
glob_targets,
resolve_output,
resolve_paths,
)
from dvc.utils.collections import ensure_list, validate from dvc.utils.collections import ensure_list, validate
from . import locked from . import locked
@@ -108,9 +103,7 @@ def translate_graph_error(stages: Stages) -> Iterator[None]:
def progress_iter(stages: Stages) -> Iterator["Stage"]: def progress_iter(stages: Stages) -> Iterator["Stage"]:
total = len(stages) total = len(stages)
desc = "Adding..." desc = "Adding..."
with ui.progress( with ui.progress(stages, total=total, desc=desc, unit="file", leave=True) as pbar:
stages, total=total, desc=desc, unit="file", leave=True
) as pbar:
if total == 1: if total == 1:
pbar.bar_format = desc pbar.bar_format = desc
pbar.refresh() pbar.refresh()
+3 -9
View File
@@ -49,15 +49,11 @@ def brancher( # noqa: E302
repo_root_parts = () repo_root_parts = ()
if self.fs.path.isin(self.root_dir, self.scm.root_dir): if self.fs.path.isin(self.root_dir, self.scm.root_dir):
repo_root_parts = self.fs.path.relparts( repo_root_parts = self.fs.path.relparts(self.root_dir, self.scm.root_dir)
self.root_dir, self.scm.root_dir
)
cwd_parts = () cwd_parts = ()
if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir): if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir):
cwd_parts = self.fs.path.relparts( cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.scm.root_dir)
self.fs.path.getcwd(), self.scm.root_dir
)
saved_fs = self.fs saved_fs = self.fs
saved_root = self.root_dir saved_root = self.root_dir
@@ -91,9 +87,7 @@ def brancher( # noqa: E302
self.root_dir = self.fs.path.join("/", *repo_root_parts) self.root_dir = self.fs.path.join("/", *repo_root_parts)
if cwd_parts: if cwd_parts:
cwd = self.fs.path.join( # type: ignore[unreachable] cwd = self.fs.path.join("/", *cwd_parts) # type: ignore[unreachable]
"/", *cwd_parts
)
self.fs.path.chdir(cwd) self.fs.path.chdir(cwd)
# ignore revs that don't contain repo root # ignore revs that don't contain repo root
+3 -12
View File
@@ -2,11 +2,7 @@ import logging
import os import os
from typing import TYPE_CHECKING, Dict, List, Set from typing import TYPE_CHECKING, Dict, List, Set
from dvc.exceptions import ( from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit, NoOutputOrStageError
CheckoutError,
CheckoutErrorSuggestGit,
NoOutputOrStageError,
)
from dvc.utils import relpath from dvc.utils import relpath
from . import locked from . import locked
@@ -35,18 +31,13 @@ def _remove_unused_links(repo):
def get_all_files_numbers(pairs): def get_all_files_numbers(pairs):
return sum( return sum(stage.get_all_files_number(filter_info) for stage, filter_info in pairs)
stage.get_all_files_number(filter_info) for stage, filter_info in pairs
)
def _collect_pairs( def _collect_pairs(
self: "Repo", targets, with_deps: bool, recursive: bool self: "Repo", targets, with_deps: bool, recursive: bool
) -> Set["StageInfo"]: ) -> Set["StageInfo"]:
from dvc.stage.exceptions import ( from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
StageFileBadNameError,
StageFileDoesNotExistError,
)
pairs: Set["StageInfo"] = set() pairs: Set["StageInfo"] = set()
for target in targets: for target in targets:
+2 -5
View File
@@ -30,8 +30,7 @@ def prompt_to_commit(stage, changes, force=False):
if not (force or prompt.confirm(_prepare_message(stage, changes))): if not (force or prompt.confirm(_prepare_message(stage, changes))):
raise StageCommitError( raise StageCommitError(
"unable to commit changed {}. Use `-f|--force` to " f"unable to commit changed {stage}. Use `-f|--force` to force."
"force.".format(stage)
) )
@@ -61,8 +60,6 @@ def commit(
if any(changes): if any(changes):
prompt_to_commit(stage, changes, force=force) prompt_to_commit(stage, changes, force=force)
stage.save(allow_missing=allow_missing) stage.save(allow_missing=allow_missing)
stage.commit( stage.commit(filter_info=stage_info.filter_info, allow_missing=allow_missing)
filter_info=stage_info.filter_info, allow_missing=allow_missing
)
stage.dump(update_pipeline=False) stage.dump(update_pipeline=False)
return [s.stage for s in stages_info] return [s.stage for s in stages_info]
+4 -11
View File
@@ -112,9 +112,7 @@ class GitInfo(TypedDict, total=False):
is_dirty: bool is_dirty: bool
def _git_info( def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitInfo:
scm: Union["Git", "NoSCM"], untracked_files: str = "all"
) -> GitInfo:
from scmrepo.exceptions import SCMError from scmrepo.exceptions import SCMError
from dvc.scm import NoSCM from dvc.scm import NoSCM
@@ -145,9 +143,7 @@ def _git_info(
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]: def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
from .index import build_data_index from .index import build_data_index
workspace = build_data_index( workspace = build_data_index(repo.index, repo.root_dir, repo.fs, compute_hash=True)
repo.index, repo.root_dir, repo.fs, compute_hash=True
)
return _diff( return _diff(
repo.index.data["repo"], repo.index.data["repo"],
@@ -180,9 +176,7 @@ class Status(TypedDict):
git: GitInfo git: GitInfo
def _transform_git_paths_to_dvc( def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
repo: "Repo", files: Iterable[str]
) -> List[str]:
"""Transform files rel. to Git root to DVC root, and drop outside files.""" """Transform files rel. to Git root to DVC root, and drop outside files."""
rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/") rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
@@ -237,8 +231,7 @@ def ls(
) -> Iterator[Dict[str, Any]]: ) -> Iterator[Dict[str, Any]]:
targets = targets or [None] targets = targets or [None]
pairs = chain.from_iterable( pairs = chain.from_iterable(
repo.stage.collect_granular(target, recursive=recursive) repo.stage.collect_granular(target, recursive=recursive) for target in targets
for target in targets
) )
for stage, filter_info in pairs: for stage, filter_info in pairs:
for out in stage.filter_outs(filter_info): for out in stage.filter_outs(filter_info):
+7 -19
View File
@@ -63,17 +63,13 @@ def diff(
targets_paths = None targets_paths = None
if targets: if targets:
# convert targets to paths, and capture any missing targets # convert targets to paths, and capture any missing targets
targets_paths, missing_targets[rev] = _targets_to_paths( targets_paths, missing_targets[rev] = _targets_to_paths(dvcfs, targets)
dvcfs, targets
)
results[rev] = _paths_checksums(self, targets_paths) results[rev] = _paths_checksums(self, targets_paths)
if targets: if targets:
# check for overlapping missing targets between a_rev and b_rev # check for overlapping missing targets between a_rev and b_rev
for target in set(missing_targets[a_rev]) & set( for target in set(missing_targets[a_rev]) & set(missing_targets[b_rev]):
missing_targets[b_rev]
):
raise PathMissingError(target, self) raise PathMissingError(target, self)
old = results[a_rev] old = results[a_rev]
@@ -112,9 +108,7 @@ def diff(
if old[path] != new[path] if old[path] != new[path]
], ],
"renamed": renamed, "renamed": renamed,
"not in cache": [ "not in cache": [{"path": path, "hash": old[path]} for path in missing],
{"path": path, "hash": old[path]} for path in missing
],
} }
return ret if any(ret.values()) else {} return ret if any(ret.values()) else {}
@@ -162,8 +156,7 @@ def _output_paths( # noqa: C901
continue continue
yield_output = targets is None or any( yield_output = targets is None or any(
output.fs.path.isin_or_eq(output.fs_path, target) output.fs.path.isin_or_eq(output.fs_path, target) for target in targets
for target in targets
) )
obj: Optional["HashFile"] obj: Optional["HashFile"]
@@ -192,15 +185,12 @@ def _output_paths( # noqa: C901
or ( or (
isinstance(targets, list) isinstance(targets, list)
and any( and any(
output.fs.path.isin(target, output.fs_path) output.fs.path.isin(target, output.fs_path) for target in targets
for target in targets
) )
) )
): ):
obj = cast("Tree", obj) obj = cast("Tree", obj)
yield from _dir_output_paths( yield from _dir_output_paths(output.fs, output.fs_path, obj, targets)
output.fs, output.fs_path, obj, targets
)
def _dir_output_paths( def _dir_output_paths(
@@ -277,8 +267,6 @@ def _calculate_renamed(
continue continue
old_path = old_paths.pop(index) old_path = old_paths.pop(index)
renamed.append( renamed.append({"path": {"old": old_path, "new": path}, "hash": path_hash})
{"path": {"old": old_path, "new": path}, "hash": path_hash}
)
return renamed return renamed
+5 -14
View File
@@ -279,20 +279,15 @@ class Experiments:
branch: Optional[str] = None branch: Optional[str] = None
try: try:
allow_multiple = bool(kwargs.get("params", None)) allow_multiple = bool(kwargs.get("params", None))
branch = self.get_branch_by_rev( branch = self.get_branch_by_rev(resume_rev, allow_multiple=allow_multiple)
resume_rev, allow_multiple=allow_multiple
)
if not branch: if not branch:
raise DvcException( raise DvcException(
"Could not find checkpoint experiment " f"Could not find checkpoint experiment '{resume_rev[:7]}'"
f"'{resume_rev[:7]}'"
) )
baseline_rev = self._get_baseline(branch) baseline_rev = self._get_baseline(branch)
except MultipleBranchError as exc: except MultipleBranchError as exc:
baselines = { baselines = {
info.baseline_sha info.baseline_sha for info in exc.ref_infos if info.baseline_sha
for info in exc.ref_infos
if info.baseline_sha
} }
if len(baselines) == 1: if len(baselines) == 1:
baseline_rev = baselines.pop() baseline_rev = baselines.pop()
@@ -341,9 +336,7 @@ class Experiments:
self.scm.remove_ref(EXEC_APPLY) self.scm.remove_ref(EXEC_APPLY)
@unlocked_repo @unlocked_repo
def _reproduce_queue( def _reproduce_queue(self, queue: "BaseStashQueue", **kwargs) -> Dict[str, str]:
self, queue: "BaseStashQueue", **kwargs
) -> Dict[str, str]:
"""Reproduce queued experiments. """Reproduce queued experiments.
Arguments: Arguments:
@@ -416,9 +409,7 @@ class Experiments:
""" """
result: Dict[str, Optional[str]] = {} result: Dict[str, Optional[str]] = {}
exclude = f"{EXEC_NAMESPACE}/*" exclude = f"{EXEC_NAMESPACE}/*"
ref_dict = self.scm.describe( ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude)
revs, base=EXPS_NAMESPACE, exclude=exclude
)
for rev in revs: for rev in revs:
name: Optional[str] = None name: Optional[str] = None
ref = ref_dict[rev] ref = ref_dict[rev]
+5 -8
View File
@@ -8,11 +8,7 @@ from dvc.repo.scm_context import scm_context
from dvc.scm import Git from dvc.scm import Git
from dvc.utils.fs import remove from dvc.utils.fs import remove
from .exceptions import ( from .exceptions import ApplyConflictError, BaselineMismatchError, InvalidExpRevError
ApplyConflictError,
BaselineMismatchError,
InvalidExpRevError,
)
from .executor.base import BaseExecutor from .executor.base import BaseExecutor
from .refs import EXEC_APPLY from .refs import EXEC_APPLY
@@ -43,7 +39,9 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901
( (
exp_ref_info, exp_ref_info,
queue_entry, queue_entry,
) = exps.celery_queue.get_ref_and_entry_by_names(rev)[rev] ) = exps.celery_queue.get_ref_and_entry_by_names(
rev
)[rev]
baseline_sha = repo.scm.get_rev() baseline_sha = repo.scm.get_rev()
if exp_ref_info: if exp_ref_info:
if baseline_sha != exp_ref_info.baseline_sha: if baseline_sha != exp_ref_info.baseline_sha:
@@ -79,8 +77,7 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901
repo.scm.set_ref(EXEC_APPLY, exp_rev) repo.scm.set_ref(EXEC_APPLY, exp_rev)
logger.info( logger.info(
"Changes for experiment '%s' have been applied to your current " "Changes for experiment '%s' have been applied to your current workspace.",
"workspace.",
rev, rev,
) )
+6 -6
View File
@@ -33,9 +33,11 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs):
break break
if not ref_info: if not ref_info:
msg = [ msg = [
f"Ambiguous experiment name '{exp_rev}' can refer to " (
"multiple experiments. To create a branch use a full " f"Ambiguous experiment name '{exp_rev}' can refer to "
"experiment ref:", "multiple experiments. To create a branch use a full "
"experiment ref:"
),
"", "",
] ]
msg.extend([str(info) for info in ref_infos]) msg.extend([str(info) for info in ref_infos])
@@ -46,9 +48,7 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs):
branch_ref = f"refs/heads/{branch_name}" branch_ref = f"refs/heads/{branch_name}"
if repo.scm.get_ref(branch_ref): if repo.scm.get_ref(branch_ref):
raise InvalidArgumentError( raise InvalidArgumentError(f"Git branch '{branch_name}' already exists.")
f"Git branch '{branch_name}' already exists."
)
target = repo.scm.get_ref(str(ref_info)) target = repo.scm.get_ref(str(ref_info))
repo.scm.set_ref( repo.scm.set_ref(
+1 -3
View File
@@ -23,9 +23,7 @@ def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs):
rev = resolve_rev(repo.scm, b_rev) rev = resolve_rev(repo.scm, b_rev)
new = collect_experiment_commit(repo, rev, param_deps=param_deps) new = collect_experiment_commit(repo, rev, param_deps=param_deps)
else: else:
new = collect_experiment_commit( new = collect_experiment_commit(repo, "workspace", param_deps=param_deps)
repo, "workspace", param_deps=param_deps
)
with_unchanged = kwargs.pop("all", False) with_unchanged = kwargs.pop("all", False)

Some files were not shown because too many files have changed in this diff Show More