mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
update black and change line-length to 88 (#8953)
This commit is contained in:
@@ -11,7 +11,7 @@ ignore=
|
||||
P1
|
||||
# use `!r` instead of surrounding by quotes
|
||||
B028
|
||||
max_line_length = 79
|
||||
max_line_length = 88
|
||||
max-complexity = 10
|
||||
select = B,C,E,F,W,T4,B902,T,P,TYP
|
||||
show_source = true
|
||||
|
||||
@@ -35,7 +35,7 @@ repos:
|
||||
args:
|
||||
- --py38-plus
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 22.12.0
|
||||
rev: 23.1.0
|
||||
hooks:
|
||||
- id: black
|
||||
- repo: https://github.com/PyCQA/isort
|
||||
|
||||
+6
-18
@@ -16,9 +16,7 @@ def viztracer_profile(
|
||||
try:
|
||||
import viztracer # pylint: disable=import-error
|
||||
except ImportError:
|
||||
print( # noqa: T201
|
||||
"Failed to run profiler, viztracer is not installed"
|
||||
)
|
||||
print("Failed to run profiler, viztracer is not installed") # noqa: T201
|
||||
yield
|
||||
return
|
||||
|
||||
@@ -80,9 +78,7 @@ def instrument(html_output=False):
|
||||
try:
|
||||
from pyinstrument import Profiler # pylint: disable=import-error
|
||||
except ImportError:
|
||||
print( # noqa: T201
|
||||
"Failed to run profiler, pyinstrument is not installed"
|
||||
)
|
||||
print("Failed to run profiler, pyinstrument is not installed") # noqa: T201
|
||||
yield
|
||||
return
|
||||
|
||||
@@ -194,9 +190,7 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs):
|
||||
path_func = _get_path_func("viztracer", "json")
|
||||
depth = kw.get("viztracer_depth") or -1
|
||||
log_async = kw.get("viztracer_async") or False
|
||||
prof = viztracer_profile(
|
||||
path=path_func, depth=depth, log_async=log_async
|
||||
)
|
||||
prof = viztracer_profile(path=path_func, depth=depth, log_async=log_async)
|
||||
stack.enter_context(prof)
|
||||
yield
|
||||
|
||||
@@ -204,12 +198,8 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs):
|
||||
def add_debugging_flags(parser):
|
||||
from argparse import SUPPRESS
|
||||
|
||||
parser.add_argument(
|
||||
"--cprofile", action="store_true", default=False, help=SUPPRESS
|
||||
)
|
||||
parser.add_argument(
|
||||
"--yappi", action="store_true", default=False, help=SUPPRESS
|
||||
)
|
||||
parser.add_argument("--cprofile", action="store_true", default=False, help=SUPPRESS)
|
||||
parser.add_argument("--yappi", action="store_true", default=False, help=SUPPRESS)
|
||||
parser.add_argument(
|
||||
"--yappi-separate-threads",
|
||||
action="store_true",
|
||||
@@ -224,9 +214,7 @@ def add_debugging_flags(parser):
|
||||
"--viztracer-async", action="store_true", default=False, help=SUPPRESS
|
||||
)
|
||||
parser.add_argument("--cprofile-dump", help=SUPPRESS)
|
||||
parser.add_argument(
|
||||
"--pdb", action="store_true", default=False, help=SUPPRESS
|
||||
)
|
||||
parser.add_argument("--pdb", action="store_true", default=False, help=SUPPRESS)
|
||||
parser.add_argument(
|
||||
"--instrument", action="store_true", default=False, help=SUPPRESS
|
||||
)
|
||||
|
||||
+2
-6
@@ -34,9 +34,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
|
||||
|
||||
class _OpenContextManager(GCM):
|
||||
def __init__(
|
||||
self, func, args, kwds
|
||||
): # pylint: disable=super-init-not-called
|
||||
def __init__(self, func, args, kwds): # pylint: disable=super-init-not-called
|
||||
self.gen = func(*args, **kwds)
|
||||
self.func, self.args, self.kwds = ( # type: ignore[assignment]
|
||||
func,
|
||||
@@ -45,9 +43,7 @@ class _OpenContextManager(GCM):
|
||||
)
|
||||
|
||||
def __getattr__(self, name):
|
||||
raise AttributeError(
|
||||
"dvc.api.open() should be used in a with statement."
|
||||
)
|
||||
raise AttributeError("dvc.api.open() should be used in a with statement.")
|
||||
|
||||
|
||||
def open( # noqa, pylint: disable=redefined-builtin
|
||||
|
||||
+9
-10
@@ -56,8 +56,10 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
|
||||
|
||||
if exc.errno == errno.EMFILE:
|
||||
logger.exception(
|
||||
(
|
||||
"too many open files, please visit "
|
||||
"%s to see how to handle this problem",
|
||||
"%s to see how to handle this problem"
|
||||
),
|
||||
error_link("many-files"),
|
||||
extra={"tb_only": True},
|
||||
)
|
||||
@@ -80,7 +82,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
|
||||
if cmd:
|
||||
link = format_link("https://dvc.org/doc/install")
|
||||
hint = (
|
||||
f"To install dvc with those dependencies, run:\n"
|
||||
"To install dvc with those dependencies, run:\n"
|
||||
"\n"
|
||||
f"\t{cmd}\n"
|
||||
"\n"
|
||||
@@ -91,8 +93,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
|
||||
hint = f"\nPlease report this bug to {link}. Thank you!"
|
||||
|
||||
logger.exception(
|
||||
"URL '%s' is supported but requires these missing "
|
||||
"dependencies: %s. %s",
|
||||
"URL '%s' is supported but requires these missing dependencies: %s. %s",
|
||||
exc.url,
|
||||
exc.missing_deps,
|
||||
hint,
|
||||
@@ -118,9 +119,11 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
|
||||
|
||||
directory = relpath(exc.directory)
|
||||
logger.exception(
|
||||
(
|
||||
"Could not open pickled '%s' cache.\n"
|
||||
"Remove the '%s' directory and then retry this command."
|
||||
"\nSee %s for more information.",
|
||||
"\nSee %s for more information."
|
||||
),
|
||||
exc.type,
|
||||
directory,
|
||||
error_link("pickle"),
|
||||
@@ -181,11 +184,7 @@ def main(argv=None): # noqa: C901
|
||||
set_loggers_level(level)
|
||||
|
||||
if level and level <= logging.DEBUG:
|
||||
from platform import (
|
||||
platform,
|
||||
python_implementation,
|
||||
python_version,
|
||||
)
|
||||
from platform import platform, python_implementation, python_version
|
||||
|
||||
from dvc import __version__
|
||||
from dvc.utils.pkg import PKG
|
||||
|
||||
+2
-6
@@ -2,9 +2,7 @@ from argparse import _AppendAction
|
||||
|
||||
|
||||
class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
|
||||
def __call__(
|
||||
self, parser, namespace, values, option_string=None # noqa: ARG002
|
||||
):
|
||||
def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002
|
||||
from funcy import ldistinct
|
||||
|
||||
items = getattr(namespace, self.dest) or []
|
||||
@@ -13,9 +11,7 @@ class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
|
||||
|
||||
|
||||
class KeyValueArgs(_AppendAction):
|
||||
def __call__(
|
||||
self, parser, namespace, values, option_string=None # noqa: ARG002
|
||||
):
|
||||
def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002
|
||||
items = getattr(namespace, self.dest) or {}
|
||||
for value in filter(bool, values):
|
||||
key, _, value = value.partition("=")
|
||||
|
||||
+1
-3
@@ -28,7 +28,5 @@ def hide_subparsers_from_help(subparsers):
|
||||
# Need to set `add_help=False`, but avoid setting `help`
|
||||
# (not even to `argparse.SUPPPRESS`).
|
||||
# NOTE: The argument is the parent subparser, not the subcommand parser.
|
||||
cmds = [
|
||||
cmd for cmd, parser in subparsers.choices.items() if parser.add_help
|
||||
]
|
||||
cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help]
|
||||
subparsers.metavar = "{{{}}}".format(",".join(cmds))
|
||||
|
||||
+1
-4
@@ -41,10 +41,7 @@ def _add_annotating_args(parser: argparse.ArgumentParser) -> None:
|
||||
|
||||
class CmdAdd(CmdBase):
|
||||
def run(self):
|
||||
from dvc.exceptions import (
|
||||
DvcException,
|
||||
RecursiveAddingWhileUsingFilename,
|
||||
)
|
||||
from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
|
||||
|
||||
try:
|
||||
if len(self.args.targets) > 1 and self.args.file:
|
||||
|
||||
@@ -68,10 +68,12 @@ def add_parser(subparsers, parent_parser):
|
||||
)
|
||||
cache_dir_parser.add_argument(
|
||||
"value",
|
||||
help="Path to cache directory. Relative paths are resolved relative "
|
||||
help=(
|
||||
"Path to cache directory. Relative paths are resolved relative "
|
||||
"to the current directory and saved to config relative to the "
|
||||
"config file location. If no path is provided, it returns the "
|
||||
"current cache directory.",
|
||||
"current cache directory."
|
||||
),
|
||||
nargs="?",
|
||||
).complete = completion.DIR
|
||||
cache_dir_parser.set_defaults(func=CmdCacheDir)
|
||||
|
||||
@@ -62,9 +62,7 @@ class CmdCheckIgnore(CmdBase):
|
||||
raise DvcException("cannot have both `targets` and `--stdin`")
|
||||
|
||||
if self.args.non_matching and not self.args.details:
|
||||
raise DvcException(
|
||||
"`--non-matching` is only valid with `--details`"
|
||||
)
|
||||
raise DvcException("`--non-matching` is only valid with `--details`")
|
||||
|
||||
if self.args.all and not self.args.details:
|
||||
raise DvcException("`--all` is only valid with `--details`")
|
||||
@@ -80,9 +78,7 @@ class CmdCheckIgnore(CmdBase):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
ADD_HELP = (
|
||||
"Check whether files or directories are excluded due to `.dvcignore`."
|
||||
)
|
||||
ADD_HELP = "Check whether files or directories are excluded due to `.dvcignore`."
|
||||
|
||||
parser = subparsers.add_parser(
|
||||
"check-ignore",
|
||||
@@ -103,16 +99,20 @@ def add_parser(subparsers, parent_parser):
|
||||
"--all",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Include the target paths which don't match any pattern "
|
||||
"in the `--details` list.",
|
||||
help=(
|
||||
"Include the target paths which don't match any pattern "
|
||||
"in the `--details` list."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n",
|
||||
"--non-matching",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Include the target paths which don't match any pattern "
|
||||
"in the `--details` list.",
|
||||
help=(
|
||||
"Include the target paths which don't match any pattern "
|
||||
"in the `--details` list."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stdin",
|
||||
|
||||
@@ -22,9 +22,7 @@ def log_changes(stats):
|
||||
continue
|
||||
|
||||
for entry in entries:
|
||||
ui.write(
|
||||
f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t"
|
||||
)
|
||||
ui.write(f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t")
|
||||
|
||||
|
||||
class CmdCheckout(CmdBase):
|
||||
@@ -46,9 +44,7 @@ class CmdCheckout(CmdBase):
|
||||
|
||||
if self.args.summary:
|
||||
default_message = "No changes."
|
||||
msg = get_summary(
|
||||
sorted(stats.items(), key=operator.itemgetter(0))
|
||||
)
|
||||
msg = get_summary(sorted(stats.items(), key=operator.itemgetter(0)))
|
||||
ui.write(msg or default_message)
|
||||
else:
|
||||
log_changes(stats)
|
||||
|
||||
@@ -24,9 +24,7 @@ class CmdCommit(CmdBase):
|
||||
force=self.args.force,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception(
|
||||
"failed to commit%s", (" " + target) if target else ""
|
||||
)
|
||||
logger.exception("failed to commit%s", (" " + target) if target else "")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
@@ -17,9 +17,7 @@ def _name_type(value):
|
||||
match = re.match(NAME_REGEX, value)
|
||||
if not match:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"name argument should look like "
|
||||
"remote.name.option or "
|
||||
"section.option"
|
||||
"name argument should look like remote.name.option or section.option"
|
||||
)
|
||||
return (
|
||||
bool(match.group("remote")),
|
||||
@@ -70,9 +68,7 @@ class CmdConfig(CmdBaseNoRepo):
|
||||
|
||||
for level in levels:
|
||||
conf = self.config.read(level)
|
||||
prefix = self._config_file_prefix(
|
||||
self.args.show_origin, self.config, level
|
||||
)
|
||||
prefix = self._config_file_prefix(self.args.show_origin, self.config, level)
|
||||
configs = list(self._format_config(conf, prefix))
|
||||
if configs:
|
||||
ui.write("\n".join(configs))
|
||||
@@ -131,9 +127,7 @@ class CmdConfig(CmdBaseNoRepo):
|
||||
raise ConfigError(f"{name} '{section}' doesn't exist")
|
||||
|
||||
if opt and opt not in conf[section]:
|
||||
raise ConfigError(
|
||||
f"option '{opt}' doesn't exist in {name} '{section}'"
|
||||
)
|
||||
raise ConfigError(f"option '{opt}' doesn't exist in {name} '{section}'")
|
||||
|
||||
def _get_appropriate_levels(self, levels):
|
||||
if levels:
|
||||
|
||||
+5
-5
@@ -86,9 +86,7 @@ def _collect_targets(repo, target, outs):
|
||||
targets.extend([str(out) for out in stage.outs])
|
||||
continue
|
||||
|
||||
for out in outs_trie.itervalues( # noqa: B301
|
||||
prefix=repo.fs.path.parts(path)
|
||||
):
|
||||
for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)): # noqa: B301
|
||||
targets.extend(str(out))
|
||||
|
||||
return targets
|
||||
@@ -204,7 +202,9 @@ def add_parser(subparsers, parent_parser):
|
||||
dag_parser.add_argument(
|
||||
"target",
|
||||
nargs="?",
|
||||
help="Stage name or output to show pipeline for. "
|
||||
"Finds all stages in the workspace by default.",
|
||||
help=(
|
||||
"Stage name or output to show pipeline for. "
|
||||
"Finds all stages in the workspace by default."
|
||||
),
|
||||
)
|
||||
dag_parser.set_defaults(func=CmdDAG)
|
||||
|
||||
+9
-18
@@ -8,11 +8,7 @@ from funcy import chunks, compact, log_durations
|
||||
from dvc.cli import completion
|
||||
from dvc.cli.actions import CommaSeparatedArgs
|
||||
from dvc.cli.command import CmdBase
|
||||
from dvc.cli.utils import (
|
||||
append_doc_link,
|
||||
fix_subparsers,
|
||||
hide_subparsers_from_help,
|
||||
)
|
||||
from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help
|
||||
from dvc.ui import ui
|
||||
from dvc.utils import colorize
|
||||
|
||||
@@ -39,20 +35,19 @@ class CmdDataStatus(CmdBase):
|
||||
}
|
||||
HINTS = {
|
||||
"not_in_cache": ('use "dvc fetch <file>..." to download files',),
|
||||
"committed": (
|
||||
"git commit the corresponding dvc files to update the repo",
|
||||
),
|
||||
"committed": ("git commit the corresponding dvc files to update the repo",),
|
||||
"uncommitted": (
|
||||
'use "dvc commit <file>..." to track changes',
|
||||
'use "dvc checkout <file>..." to discard changes',
|
||||
),
|
||||
"untracked": (
|
||||
(
|
||||
'use "git add <file> ..." or '
|
||||
'"dvc add <file>..." to commit to git or to dvc',
|
||||
'"dvc add <file>..." to commit to git or to dvc'
|
||||
),
|
||||
),
|
||||
"git_dirty": (
|
||||
"there are {}changes not tracked by dvc, "
|
||||
'use "git status" to see',
|
||||
'there are {}changes not tracked by dvc, use "git status" to see',
|
||||
),
|
||||
}
|
||||
|
||||
@@ -96,8 +91,7 @@ class CmdDataStatus(CmdBase):
|
||||
|
||||
if isinstance(stage_status, dict):
|
||||
items = [
|
||||
": ".join([state, file])
|
||||
for file, state in stage_status.items()
|
||||
": ".join([state, file]) for file, state in stage_status.items()
|
||||
]
|
||||
else:
|
||||
items = stage_status
|
||||
@@ -180,9 +174,7 @@ class CmdDataLs(CmdBase):
|
||||
|
||||
filter_labels = set(self.args.labels)
|
||||
filter_types = set(self.args.type)
|
||||
d = ls(
|
||||
self.repo, targets=self.args.targets, recursive=self.args.recursive
|
||||
)
|
||||
d = ls(self.repo, targets=self.args.targets, recursive=self.args.recursive)
|
||||
self._show_table(
|
||||
d,
|
||||
filter_labels=filter_labels,
|
||||
@@ -205,8 +197,7 @@ def add_parser(subparsers, parent_parser):
|
||||
fix_subparsers(data_subparsers)
|
||||
|
||||
DATA_STATUS_HELP = (
|
||||
"Show changes between the last git commit, "
|
||||
"the dvcfiles and the workspace."
|
||||
"Show changes between the last git commit, the dvcfiles and the workspace."
|
||||
)
|
||||
data_status_parser = data_subparsers.add_parser(
|
||||
"status",
|
||||
|
||||
+13
-12
@@ -262,9 +262,7 @@ def add_parser(subparsers, _parent_parser):
|
||||
push_parser.set_defaults(func=CmdDataPush)
|
||||
|
||||
# Fetch
|
||||
FETCH_HELP = (
|
||||
"Download files or directories from remote storage to the cache."
|
||||
)
|
||||
FETCH_HELP = "Download files or directories from remote storage to the cache."
|
||||
|
||||
fetch_parser = subparsers.add_parser(
|
||||
"fetch",
|
||||
@@ -320,9 +318,7 @@ def add_parser(subparsers, _parent_parser):
|
||||
fetch_parser.set_defaults(func=CmdDataFetch)
|
||||
|
||||
# Status
|
||||
STATUS_HELP = (
|
||||
"Show changed stages, compare local cache and a remote storage."
|
||||
)
|
||||
STATUS_HELP = "Show changed stages, compare local cache and a remote storage."
|
||||
|
||||
status_parser = subparsers.add_parser(
|
||||
"status",
|
||||
@@ -360,24 +356,29 @@ def add_parser(subparsers, _parent_parser):
|
||||
"--all-branches",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show status of a local cache compared to a remote repository "
|
||||
"for all branches.",
|
||||
help=(
|
||||
"Show status of a local cache compared to a remote repository "
|
||||
"for all branches."
|
||||
),
|
||||
)
|
||||
status_parser.add_argument(
|
||||
"-T",
|
||||
"--all-tags",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show status of a local cache compared to a remote repository "
|
||||
"for all tags.",
|
||||
help=(
|
||||
"Show status of a local cache compared to a remote repository for all tags."
|
||||
),
|
||||
)
|
||||
status_parser.add_argument(
|
||||
"-A",
|
||||
"--all-commits",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Show status of a local cache compared to a remote repository "
|
||||
"for all commits.",
|
||||
help=(
|
||||
"Show status of a local cache compared to a remote repository "
|
||||
"for all commits."
|
||||
),
|
||||
)
|
||||
status_parser.add_argument(
|
||||
"-d",
|
||||
|
||||
+8
-13
@@ -112,9 +112,7 @@ class CmdDiff(CmdBase):
|
||||
return
|
||||
|
||||
states_summary = ", ".join(
|
||||
f"{summary[state]} {state}"
|
||||
for state in states
|
||||
if summary[state] > 0
|
||||
f"{summary[state]} {state}" for state in states if summary[state] > 0
|
||||
)
|
||||
ui.write("files summary:", states_summary)
|
||||
|
||||
@@ -122,9 +120,7 @@ class CmdDiff(CmdBase):
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
try:
|
||||
diff = self.repo.diff(
|
||||
self.args.a_rev, self.args.b_rev, self.args.targets
|
||||
)
|
||||
diff = self.repo.diff(self.args.a_rev, self.args.b_rev, self.args.targets)
|
||||
show_hash = self.args.show_hash
|
||||
hide_missing = self.args.b_rev or self.args.hide_missing
|
||||
if hide_missing:
|
||||
@@ -133,9 +129,11 @@ class CmdDiff(CmdBase):
|
||||
for key, entries in diff.items():
|
||||
entries = sorted(
|
||||
entries,
|
||||
key=lambda entry: entry["path"]["old"]
|
||||
key=lambda entry: (
|
||||
entry["path"]["old"]
|
||||
if isinstance(entry["path"], dict)
|
||||
else entry["path"],
|
||||
else entry["path"]
|
||||
),
|
||||
)
|
||||
if not show_hash:
|
||||
for entry in entries:
|
||||
@@ -170,10 +168,7 @@ def add_parser(subparsers, parent_parser):
|
||||
diff_parser.add_argument(
|
||||
"--targets",
|
||||
nargs="*",
|
||||
help=(
|
||||
"Specific DVC-tracked files to compare. "
|
||||
"Accepts one or more file paths."
|
||||
),
|
||||
help="Specific DVC-tracked files to compare. Accepts one or more file paths.",
|
||||
metavar="<paths>",
|
||||
).complete = completion.FILE
|
||||
diff_parser.add_argument(
|
||||
@@ -184,7 +179,7 @@ def add_parser(subparsers, parent_parser):
|
||||
)
|
||||
diff_parser.add_argument(
|
||||
"b_rev",
|
||||
help=("New Git commit to compare (defaults to the current workspace)"),
|
||||
help="New Git commit to compare (defaults to the current workspace)",
|
||||
nargs="?",
|
||||
)
|
||||
diff_parser.add_argument(
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
import argparse
|
||||
|
||||
from dvc.cli.utils import (
|
||||
append_doc_link,
|
||||
fix_subparsers,
|
||||
hide_subparsers_from_help,
|
||||
)
|
||||
from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help
|
||||
from dvc.commands.experiments import (
|
||||
apply,
|
||||
branch,
|
||||
@@ -54,8 +50,7 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
experiments_subparsers = experiments_parser.add_subparsers(
|
||||
dest="cmd",
|
||||
help="Use `dvc experiments CMD --help` to display "
|
||||
"command-specific help.",
|
||||
help="Use `dvc experiments CMD --help` to display command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(experiments_subparsers)
|
||||
|
||||
@@ -10,19 +10,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class CmdExperimentsApply(CmdBase):
|
||||
def run(self):
|
||||
|
||||
self.repo.experiments.apply(
|
||||
self.args.experiment, force=self.args.force
|
||||
)
|
||||
self.repo.experiments.apply(self.args.experiment, force=self.args.force)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(experiments_subparsers, parent_parser):
|
||||
|
||||
EXPERIMENTS_APPLY_HELP = (
|
||||
"Apply the changes from an experiment to your workspace."
|
||||
)
|
||||
EXPERIMENTS_APPLY_HELP = "Apply the changes from an experiment to your workspace."
|
||||
experiments_apply_parser = experiments_subparsers.add_parser(
|
||||
"apply",
|
||||
parents=[parent_parser],
|
||||
|
||||
@@ -9,14 +9,12 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class CmdExperimentsBranch(CmdBase):
|
||||
def run(self):
|
||||
|
||||
self.repo.experiments.branch(self.args.experiment, self.args.branch)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(experiments_subparsers, parent_parser):
|
||||
|
||||
EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch."
|
||||
experiments_branch_parser = experiments_subparsers.add_parser(
|
||||
"branch",
|
||||
@@ -28,7 +26,5 @@ def add_parser(experiments_subparsers, parent_parser):
|
||||
experiments_branch_parser.add_argument(
|
||||
"experiment", help="Experiment to be promoted."
|
||||
)
|
||||
experiments_branch_parser.add_argument(
|
||||
"branch", help="Name for the new Git branch"
|
||||
)
|
||||
experiments_branch_parser.add_argument("branch", help="Name for the new Git branch")
|
||||
experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)
|
||||
|
||||
@@ -13,7 +13,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class CmdExperimentsDiff(CmdBase):
|
||||
def run(self):
|
||||
|
||||
try:
|
||||
diff = self.repo.experiments.diff(
|
||||
a_rev=self.args.a_rev,
|
||||
@@ -53,10 +52,7 @@ class CmdExperimentsDiff(CmdBase):
|
||||
|
||||
|
||||
def add_parser(experiments_subparsers, parent_parser):
|
||||
|
||||
EXPERIMENTS_DIFF_HELP = (
|
||||
"Show changes between experiments in the DVC repository."
|
||||
)
|
||||
EXPERIMENTS_DIFF_HELP = "Show changes between experiments in the DVC repository."
|
||||
|
||||
experiments_diff_parser = experiments_subparsers.add_parser(
|
||||
"diff",
|
||||
|
||||
@@ -9,10 +9,7 @@ class CmdExecutorRun(CmdBaseNoRepo):
|
||||
"""Run an experiment executor."""
|
||||
|
||||
def run(self):
|
||||
from dvc.repo.experiments.executor.base import (
|
||||
BaseExecutor,
|
||||
ExecutorInfo,
|
||||
)
|
||||
from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo
|
||||
from dvc.utils.serialize import load_json
|
||||
|
||||
info = ExecutorInfo.from_dict(load_json(self.args.infofile))
|
||||
|
||||
@@ -41,10 +41,7 @@ class CmdExperimentsGC(CmdBase):
|
||||
msg += " and all git tags"
|
||||
|
||||
if self.args.commit_date:
|
||||
msg += (
|
||||
" and all git commits before date "
|
||||
f"{self.args.commit_date}"
|
||||
)
|
||||
msg += f" and all git commits before date {self.args.commit_date}"
|
||||
msg += " of the current repo."
|
||||
|
||||
if self.args.queued:
|
||||
@@ -79,11 +76,9 @@ class CmdExperimentsGC(CmdBase):
|
||||
|
||||
|
||||
def add_parser(experiments_subparsers, parent_parser):
|
||||
|
||||
EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
|
||||
EXPERIMENTS_GC_DESCRIPTION = (
|
||||
"Removes all experiments which are not derived from the specified"
|
||||
"Git revisions."
|
||||
"Removes all experiments which are not derived from the specifiedGit revisions."
|
||||
)
|
||||
experiments_gc_parser = experiments_subparsers.add_parser(
|
||||
"gc",
|
||||
|
||||
@@ -92,9 +92,7 @@ class CmdExperimentsInit(CmdBase):
|
||||
|
||||
if new_out_dirs:
|
||||
out_dirs_paths = humanize.join(map(path_fmt, new_out_dirs))
|
||||
ui.write(
|
||||
f"Creating output directories: {out_dirs_paths}", styled=True
|
||||
)
|
||||
ui.write(f"Creating output directories: {out_dirs_paths}", styled=True)
|
||||
|
||||
ui.write(
|
||||
f"Creating [b]{self.args.name}[/b] stage in [green]dvc.yaml[/]",
|
||||
@@ -120,7 +118,6 @@ class CmdExperimentsInit(CmdBase):
|
||||
|
||||
|
||||
def add_parser(experiments_subparsers, parent_parser):
|
||||
|
||||
EXPERIMENTS_INIT_HELP = "Quickly setup any project to use experiments."
|
||||
experiments_init_parser = experiments_subparsers.add_parser(
|
||||
"init",
|
||||
@@ -167,35 +164,47 @@ def add_parser(experiments_subparsers, parent_parser):
|
||||
)
|
||||
experiments_init_parser.add_argument(
|
||||
"--code",
|
||||
help="Path to the source file or directory "
|
||||
help=(
|
||||
"Path to the source file or directory "
|
||||
"which your experiments depend"
|
||||
f" (default: {CmdExperimentsInit.CODE})",
|
||||
f" (default: {CmdExperimentsInit.CODE})"
|
||||
),
|
||||
)
|
||||
experiments_init_parser.add_argument(
|
||||
"--data",
|
||||
help="Path to the data file or directory "
|
||||
help=(
|
||||
"Path to the data file or directory "
|
||||
"which your experiments depend"
|
||||
f" (default: {CmdExperimentsInit.DATA})",
|
||||
f" (default: {CmdExperimentsInit.DATA})"
|
||||
),
|
||||
)
|
||||
experiments_init_parser.add_argument(
|
||||
"--models",
|
||||
help="Path to the model file or directory for your experiments"
|
||||
f" (default: {CmdExperimentsInit.MODELS})",
|
||||
help=(
|
||||
"Path to the model file or directory for your experiments"
|
||||
f" (default: {CmdExperimentsInit.MODELS})"
|
||||
),
|
||||
)
|
||||
experiments_init_parser.add_argument(
|
||||
"--params",
|
||||
help="Path to the parameters file for your experiments"
|
||||
f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})",
|
||||
help=(
|
||||
"Path to the parameters file for your experiments"
|
||||
f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})"
|
||||
),
|
||||
)
|
||||
experiments_init_parser.add_argument(
|
||||
"--metrics",
|
||||
help="Path to the metrics file for your experiments"
|
||||
f" (default: {CmdExperimentsInit.DEFAULT_METRICS})",
|
||||
help=(
|
||||
"Path to the metrics file for your experiments"
|
||||
f" (default: {CmdExperimentsInit.DEFAULT_METRICS})"
|
||||
),
|
||||
)
|
||||
experiments_init_parser.add_argument(
|
||||
"--plots",
|
||||
help="Path to the plots file or directory for your experiments"
|
||||
f" (default: {CmdExperimentsInit.PLOTS})",
|
||||
help=(
|
||||
"Path to the plots file or directory for your experiments"
|
||||
f" (default: {CmdExperimentsInit.PLOTS})"
|
||||
),
|
||||
)
|
||||
experiments_init_parser.add_argument(
|
||||
"--live",
|
||||
|
||||
@@ -11,9 +11,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class CmdExperimentsPull(CmdBase):
|
||||
def raise_error_if_all_disabled(self):
|
||||
if not any(
|
||||
[self.args.experiment, self.args.all_commits, self.args.rev]
|
||||
):
|
||||
if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
|
||||
raise InvalidArgumentError(
|
||||
"Either provide an `experiment` argument, or use the "
|
||||
"`--rev` or `--all-commits` flag."
|
||||
@@ -74,10 +72,7 @@ def add_parser(experiments_subparsers, parent_parser):
|
||||
"--no-cache",
|
||||
action="store_false",
|
||||
dest="pull_cache",
|
||||
help=(
|
||||
"Do not pull cached outputs for this experiment from DVC remote "
|
||||
"storage."
|
||||
),
|
||||
help="Do not pull cached outputs for this experiment from DVC remote storage.",
|
||||
)
|
||||
experiments_pull_parser.add_argument(
|
||||
"-r",
|
||||
@@ -92,8 +87,7 @@ def add_parser(experiments_subparsers, parent_parser):
|
||||
type=int,
|
||||
metavar="<number>",
|
||||
help=(
|
||||
"Number of jobs to run simultaneously when pulling from DVC "
|
||||
"remote storage."
|
||||
"Number of jobs to run simultaneously when pulling from DVC remote storage."
|
||||
),
|
||||
)
|
||||
experiments_pull_parser.add_argument(
|
||||
|
||||
@@ -12,16 +12,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class CmdExperimentsPush(CmdBase):
|
||||
def raise_error_if_all_disabled(self):
|
||||
if not any(
|
||||
[self.args.experiment, self.args.all_commits, self.args.rev]
|
||||
):
|
||||
if not any([self.args.experiment, self.args.all_commits, self.args.rev]):
|
||||
raise InvalidArgumentError(
|
||||
"Either provide an `experiment` argument, or use the "
|
||||
"`--rev` or `--all-commits` flag."
|
||||
)
|
||||
|
||||
def run(self):
|
||||
|
||||
self.raise_error_if_all_disabled()
|
||||
|
||||
pushed_exps = self.repo.experiments.push(
|
||||
@@ -47,8 +44,10 @@ class CmdExperimentsPush(CmdBase):
|
||||
if not self.args.push_cache:
|
||||
ui.write(
|
||||
"To push cached outputs",
|
||||
(
|
||||
"for this experiment to DVC remote storage,"
|
||||
"re-run this command without '--no-cache'.",
|
||||
"re-run this command without '--no-cache'."
|
||||
),
|
||||
)
|
||||
|
||||
return 0
|
||||
@@ -76,10 +75,7 @@ def add_parser(experiments_subparsers, parent_parser):
|
||||
"--no-cache",
|
||||
action="store_false",
|
||||
dest="push_cache",
|
||||
help=(
|
||||
"Do not push cached outputs for this experiment to DVC remote "
|
||||
"storage."
|
||||
),
|
||||
help="Do not push cached outputs for this experiment to DVC remote storage.",
|
||||
)
|
||||
experiments_push_parser.add_argument(
|
||||
"-r",
|
||||
@@ -93,10 +89,7 @@ def add_parser(experiments_subparsers, parent_parser):
|
||||
"--jobs",
|
||||
type=int,
|
||||
metavar="<number>",
|
||||
help=(
|
||||
"Number of jobs to run simultaneously when pushing to DVC remote "
|
||||
"storage."
|
||||
),
|
||||
help="Number of jobs to run simultaneously when pushing to DVC remote storage.",
|
||||
)
|
||||
experiments_push_parser.add_argument(
|
||||
"--run-cache",
|
||||
|
||||
@@ -24,7 +24,6 @@ class CmdExperimentsRemove(CmdBase):
|
||||
)
|
||||
|
||||
def run(self):
|
||||
|
||||
self.check_arguments()
|
||||
|
||||
removed_list = self.repo.experiments.remove(
|
||||
|
||||
@@ -17,13 +17,10 @@ class CmdExperimentsRun(CmdRepro):
|
||||
|
||||
if self.args.checkpoint_resume:
|
||||
if self.args.reset:
|
||||
raise InvalidArgumentError(
|
||||
"--reset and --rev are mutually exclusive."
|
||||
)
|
||||
raise InvalidArgumentError("--reset and --rev are mutually exclusive.")
|
||||
if not (self.args.queue or self.args.tmp_dir):
|
||||
raise InvalidArgumentError(
|
||||
"--rev can only be used in conjunction with "
|
||||
"--queue or --temp."
|
||||
"--rev can only be used in conjunction with --queue or --temp."
|
||||
)
|
||||
|
||||
if self.args.reset:
|
||||
@@ -51,7 +48,6 @@ class CmdExperimentsRun(CmdRepro):
|
||||
|
||||
|
||||
def add_parser(experiments_subparsers, parent_parser):
|
||||
|
||||
EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
|
||||
experiments_run_parser = experiments_subparsers.add_parser(
|
||||
"run",
|
||||
|
||||
@@ -11,7 +11,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class CmdExperimentsSave(CmdBase):
|
||||
def run(self):
|
||||
|
||||
try:
|
||||
ref = self.repo.experiments.save(
|
||||
name=self.args.name,
|
||||
|
||||
@@ -84,9 +84,7 @@ def _collect_rows( # noqa: C901
|
||||
sort_by, metric_names, param_names
|
||||
)
|
||||
reverse = sort_order == "desc"
|
||||
experiments = _sort_exp(
|
||||
experiments, sort_path, sort_name, sort_type, reverse
|
||||
)
|
||||
experiments = _sort_exp(experiments, sort_path, sort_name, sort_type, reverse)
|
||||
|
||||
new_checkpoint = True
|
||||
for i, (rev, results) in enumerate(experiments.items()):
|
||||
@@ -116,9 +114,7 @@ def _collect_rows( # noqa: C901
|
||||
typ = "baseline"
|
||||
elif tip:
|
||||
if tip == parent_tip:
|
||||
typ = (
|
||||
"checkpoint_tip" if new_checkpoint else "checkpoint_commit"
|
||||
)
|
||||
typ = "checkpoint_tip" if new_checkpoint else "checkpoint_commit"
|
||||
elif parent_rev == base_rev:
|
||||
typ = "checkpoint_base"
|
||||
else:
|
||||
@@ -135,9 +131,7 @@ def _collect_rows( # noqa: C901
|
||||
row_dict["Experiment"] = exp.get("name", "")
|
||||
row_dict["rev"] = name_rev
|
||||
row_dict["typ"] = typ
|
||||
row_dict["Created"] = format_time(
|
||||
exp.get("timestamp"), fill_value, iso
|
||||
)
|
||||
row_dict["Created"] = format_time(exp.get("timestamp"), fill_value, iso)
|
||||
row_dict["parent"] = parent
|
||||
row_dict["State"] = state
|
||||
row_dict["Executor"] = exp.get("executor", fill_value)
|
||||
@@ -371,9 +365,7 @@ def show_experiments( # noqa: C901
|
||||
|
||||
if not csv:
|
||||
merge_headers = ["Experiment", "rev", "typ", "parent"]
|
||||
td.column("Experiment")[:] = map(
|
||||
prepare_exp_id, td.as_dict(merge_headers)
|
||||
)
|
||||
td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers))
|
||||
td.drop(*merge_headers[1:])
|
||||
|
||||
headers = {
|
||||
@@ -410,9 +402,7 @@ def show_experiments( # noqa: C901
|
||||
|
||||
cols_to_drop = set()
|
||||
if drop is not None:
|
||||
cols_to_drop = {
|
||||
col for col in td.keys() if re.match(drop, col) # noqa: SIM118
|
||||
}
|
||||
cols_to_drop = {col for col in td.keys() if re.match(drop, col)} # noqa: SIM118
|
||||
if pcp:
|
||||
cols_to_drop.add("Created")
|
||||
td.drop(*cols_to_drop)
|
||||
@@ -491,9 +481,7 @@ class CmdExperimentsShow(CmdBase):
|
||||
ui.write_json(all_experiments, default=_format_json)
|
||||
else:
|
||||
precision = (
|
||||
self.args.precision or None
|
||||
if self.args.csv
|
||||
else DEFAULT_PRECISION
|
||||
self.args.precision or None if self.args.csv else DEFAULT_PRECISION
|
||||
)
|
||||
fill_value = "" if self.args.csv else FILL_VALUE
|
||||
iso = self.args.csv
|
||||
@@ -582,10 +570,7 @@ def add_parser(experiments_subparsers, parent_parser):
|
||||
)
|
||||
experiments_show_parser.add_argument(
|
||||
"--sort-order",
|
||||
help=(
|
||||
"Sort order to use with --sort-by."
|
||||
" Defaults to ascending ('asc')."
|
||||
),
|
||||
help="Sort order to use with --sort-by. Defaults to ascending ('asc').",
|
||||
choices=("asc", "desc"),
|
||||
default="asc",
|
||||
)
|
||||
|
||||
+6
-9
@@ -40,14 +40,9 @@ class CmdGC(CmdBase):
|
||||
elif self.args.all_tags:
|
||||
msg += " and all git tags"
|
||||
if self.args.commit_date:
|
||||
msg += (
|
||||
" and all git commits before date "
|
||||
f"{self.args.commit_date}"
|
||||
)
|
||||
msg += f" and all git commits before date {self.args.commit_date}"
|
||||
if self.args.rev:
|
||||
msg += (
|
||||
f" and last {self.args.num} commits from {self.args.rev}"
|
||||
)
|
||||
msg += f" and last {self.args.num} commits from {self.args.rev}"
|
||||
|
||||
if self.args.all_experiments:
|
||||
msg += " and all experiments"
|
||||
@@ -198,9 +193,11 @@ def add_parser(subparsers, parent_parser):
|
||||
dest="repos",
|
||||
type=str,
|
||||
nargs="*",
|
||||
help="Keep data files required by these projects "
|
||||
help=(
|
||||
"Keep data files required by these projects "
|
||||
"in addition to the current one. "
|
||||
"Useful if you share a single cache across repos.",
|
||||
"Useful if you share a single cache across repos."
|
||||
),
|
||||
metavar="<paths>",
|
||||
)
|
||||
gc_parser.set_defaults(func=CmdGC)
|
||||
|
||||
+5
-5
@@ -16,9 +16,7 @@ class CmdGet(CmdBaseNoRepo):
|
||||
from dvc.ui import ui
|
||||
|
||||
try:
|
||||
url = get_url(
|
||||
self.args.path, repo=self.args.url, rev=self.args.rev
|
||||
)
|
||||
url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
|
||||
ui.write(url, force=True)
|
||||
except DvcException:
|
||||
logger.exception("failed to show URL")
|
||||
@@ -85,8 +83,10 @@ def add_parser(subparsers, parent_parser):
|
||||
get_parser.add_argument(
|
||||
"--show-url",
|
||||
action="store_true",
|
||||
help="Print the storage location (URL) the target data would be "
|
||||
"downloaded from, and exit.",
|
||||
help=(
|
||||
"Print the storage location (URL) the target data would be "
|
||||
"downloaded from, and exit."
|
||||
),
|
||||
)
|
||||
get_parser.add_argument(
|
||||
"-j",
|
||||
|
||||
+4
-2
@@ -90,8 +90,10 @@ def add_parser(subparsers, parent_parser):
|
||||
"--no-download",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Create .dvc file including target data hash value(s)"
|
||||
" but do not actually download the file(s).",
|
||||
help=(
|
||||
"Create .dvc file including target data hash value(s)"
|
||||
" but do not actually download the file(s)."
|
||||
),
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"-j",
|
||||
|
||||
+13
-12
@@ -29,8 +29,10 @@ class CmdImportUrl(CmdBase):
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception(
|
||||
(
|
||||
"failed to import %s. You could also try downloading "
|
||||
"it manually, and adding it with `dvc add`.",
|
||||
"it manually, and adding it with `dvc add`."
|
||||
),
|
||||
self.args.url,
|
||||
)
|
||||
return 1
|
||||
@@ -40,9 +42,7 @@ class CmdImportUrl(CmdBase):
|
||||
def add_parser(subparsers, parent_parser):
|
||||
from .add import _add_annotating_args
|
||||
|
||||
IMPORT_HELP = (
|
||||
"Download or copy file from URL and take it under DVC control."
|
||||
)
|
||||
IMPORT_HELP = "Download or copy file from URL and take it under DVC control."
|
||||
|
||||
import_parser = subparsers.add_parser(
|
||||
"import-url",
|
||||
@@ -53,7 +53,8 @@ def add_parser(subparsers, parent_parser):
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"url",
|
||||
help="Location of the data to download. Supported URLs:\n"
|
||||
help=(
|
||||
"Location of the data to download. Supported URLs:\n"
|
||||
"/absolute/path/to/file/or/dir\n"
|
||||
"relative/path/to/file/or/dir\n"
|
||||
"C:\\\\path\\to\\file\\or\\dir\n"
|
||||
@@ -62,7 +63,8 @@ def add_parser(subparsers, parent_parser):
|
||||
"gs://bucket/path/to/file/or/dir\n"
|
||||
"hdfs://example.com/path/to/file\n"
|
||||
"ssh://example.com/absolute/path/to/file/or/dir\n"
|
||||
"remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
|
||||
"remote://remote_name/path/to/file/or/dir (see `dvc remote`)"
|
||||
),
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"out", nargs="?", help="Destination path to put files to."
|
||||
@@ -95,8 +97,10 @@ def add_parser(subparsers, parent_parser):
|
||||
"--no-download",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Create .dvc file including target data hash value(s)"
|
||||
" but do not actually download the file(s).",
|
||||
help=(
|
||||
"Create .dvc file including target data hash value(s)"
|
||||
" but do not actually download the file(s)."
|
||||
),
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"-j",
|
||||
@@ -112,10 +116,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"--version-aware",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Import using cloud versioning. Implied if the URL contains a "
|
||||
"version ID."
|
||||
),
|
||||
help="Import using cloud versioning. Implied if the URL contains a version ID.",
|
||||
)
|
||||
_add_annotating_args(import_parser)
|
||||
import_parser.set_defaults(func=CmdImportUrl)
|
||||
|
||||
@@ -75,8 +75,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"--no-scm",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Initiate DVC in directory that is "
|
||||
"not tracked by any SCM tool (e.g. Git).",
|
||||
help=(
|
||||
"Initiate DVC in directory that is not tracked by any SCM tool (e.g. Git)."
|
||||
),
|
||||
)
|
||||
init_parser.add_argument(
|
||||
"-f",
|
||||
@@ -84,8 +85,7 @@ def add_parser(subparsers, parent_parser):
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Overwrite existing '.dvc/' directory. "
|
||||
"This operation removes local cache."
|
||||
"Overwrite existing '.dvc/' directory. This operation removes local cache."
|
||||
),
|
||||
)
|
||||
init_parser.add_argument(
|
||||
|
||||
@@ -31,7 +31,9 @@ def add_parser(subparsers, parent_parser):
|
||||
"--use-pre-commit-tool",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Install DVC hooks using pre-commit "
|
||||
"(https://pre-commit.com) if it is installed.",
|
||||
help=(
|
||||
"Install DVC hooks using pre-commit "
|
||||
"(https://pre-commit.com) if it is installed."
|
||||
),
|
||||
)
|
||||
install_parser.set_defaults(func=CmdInstall)
|
||||
|
||||
+9
-28
@@ -18,7 +18,6 @@ class MachineDisabledError(ConfigError):
|
||||
|
||||
class CmdMachineConfig(CmdConfig):
|
||||
def __init__(self, args):
|
||||
|
||||
super().__init__(args)
|
||||
if not self.config["feature"].get("machine", False):
|
||||
raise MachineDisabledError
|
||||
@@ -163,14 +162,11 @@ class CmdMachineRename(CmdBase):
|
||||
all_config = self.config.load_config_to_level(None)
|
||||
if self.args.new in all_config.get("machine", {}):
|
||||
raise ConfigError(
|
||||
"Rename failed. Machine '{}' already exists.".format(
|
||||
self.args.new
|
||||
)
|
||||
f"Rename failed. Machine '{self.args.new}' already exists."
|
||||
)
|
||||
ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.")
|
||||
|
||||
def run(self):
|
||||
|
||||
self._check_before_rename()
|
||||
|
||||
with self.config.edit(self.args.level) as conf:
|
||||
@@ -209,9 +205,7 @@ class CmdMachineDefault(CmdMachineConfig):
|
||||
if self.args.unset:
|
||||
conf["core"].pop("machine", None)
|
||||
else:
|
||||
merged_conf = self.config.load_config_to_level(
|
||||
self.args.level
|
||||
)
|
||||
merged_conf = self.config.load_config_to_level(self.args.level)
|
||||
if (
|
||||
self.args.name in conf["machine"]
|
||||
or self.args.name in merged_conf["machine"]
|
||||
@@ -219,8 +213,7 @@ class CmdMachineDefault(CmdMachineConfig):
|
||||
conf["core"]["machine"] = self.args.name
|
||||
else:
|
||||
raise ConfigError(
|
||||
"default machine must be present in machine "
|
||||
"list."
|
||||
"default machine must be present in machine list."
|
||||
)
|
||||
return 0
|
||||
|
||||
@@ -251,7 +244,6 @@ class CmdMachineStatus(CmdBase):
|
||||
all_status: List[Dict],
|
||||
td: TabularData,
|
||||
):
|
||||
|
||||
if not all_status:
|
||||
row = [
|
||||
name,
|
||||
@@ -343,8 +335,7 @@ def add_parser(subparsers, parent_parser):
|
||||
"cloud",
|
||||
help="Machine cloud. See full list of supported clouds at {}".format(
|
||||
format_link(
|
||||
"https://github.com/iterative/"
|
||||
"terraform-provider-iterative#machine"
|
||||
"https://github.com/iterative/terraform-provider-iterative#machine"
|
||||
)
|
||||
),
|
||||
)
|
||||
@@ -372,9 +363,7 @@ def add_parser(subparsers, parent_parser):
|
||||
help=machine_DEFAULT_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
machine_default_parser.add_argument(
|
||||
"name", nargs="?", help="Name of the machine"
|
||||
)
|
||||
machine_default_parser.add_argument("name", nargs="?", help="Name of the machine")
|
||||
machine_default_parser.add_argument(
|
||||
"-u",
|
||||
"--unset",
|
||||
@@ -414,9 +403,7 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
machine_modify_parser.add_argument("name", help="Name of the machine")
|
||||
machine_modify_parser.add_argument(
|
||||
"option", help="Name of the option to modify."
|
||||
)
|
||||
machine_modify_parser.add_argument("option", help="Name of the option to modify.")
|
||||
machine_modify_parser.add_argument(
|
||||
"value", nargs="?", help="(optional) Value of the option."
|
||||
)
|
||||
@@ -449,9 +436,7 @@ def add_parser(subparsers, parent_parser):
|
||||
help=machine_REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
machine_remove_parser.add_argument(
|
||||
"name", help="Name of the machine to remove."
|
||||
)
|
||||
machine_remove_parser.add_argument("name", help="Name of the machine to remove.")
|
||||
machine_remove_parser.set_defaults(func=CmdMachineRemove)
|
||||
|
||||
machine_CREATE_HELP = "Create and start a machine instance."
|
||||
@@ -462,14 +447,10 @@ def add_parser(subparsers, parent_parser):
|
||||
help=machine_CREATE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
machine_create_parser.add_argument(
|
||||
"name", help="Name of the machine to create."
|
||||
)
|
||||
machine_create_parser.add_argument("name", help="Name of the machine to create.")
|
||||
machine_create_parser.set_defaults(func=CmdMachineCreate)
|
||||
|
||||
machine_STATUS_HELP = (
|
||||
"List the status of running instances for one/all machines."
|
||||
)
|
||||
machine_STATUS_HELP = "List the status of running instances for one/all machines."
|
||||
machine_status_parser = machine_subparsers.add_parser(
|
||||
"status",
|
||||
parents=[parent_parser],
|
||||
|
||||
@@ -14,9 +14,7 @@ class CmdMove(CmdBase):
|
||||
try:
|
||||
self.repo.move(self.args.src, self.args.dst)
|
||||
except DvcException:
|
||||
msg = "failed to move '{}' -> '{}'".format(
|
||||
self.args.src, self.args.dst
|
||||
)
|
||||
msg = f"failed to move '{self.args.src}' -> '{self.args.dst}'"
|
||||
logger.exception(msg)
|
||||
return 1
|
||||
return 0
|
||||
@@ -40,7 +38,5 @@ def add_parser(subparsers, parent_parser):
|
||||
move_parser.add_argument(
|
||||
"src", help="Source path to a data file or directory."
|
||||
).complete = completion.FILE
|
||||
move_parser.add_argument(
|
||||
"dst", help="Destination path."
|
||||
).complete = completion.FILE
|
||||
move_parser.add_argument("dst", help="Destination path.").complete = completion.FILE
|
||||
move_parser.set_defaults(func=CmdMove)
|
||||
|
||||
@@ -79,7 +79,7 @@ def add_parser(subparsers, parent_parser):
|
||||
params_diff_parser.add_argument(
|
||||
"b_rev",
|
||||
nargs="?",
|
||||
help=("New Git commit to compare (defaults to the current workspace)"),
|
||||
help="New Git commit to compare (defaults to the current workspace)",
|
||||
)
|
||||
params_diff_parser.add_argument(
|
||||
"--targets",
|
||||
|
||||
+12
-31
@@ -18,9 +18,7 @@ logger = logging.getLogger(__name__)
|
||||
def _show_json(renderers, split=False):
|
||||
from dvc.render.convert import to_json
|
||||
|
||||
result = {
|
||||
renderer.name: to_json(renderer, split) for renderer in renderers
|
||||
}
|
||||
result = {renderer.name: to_json(renderer, split) for renderer in renderers}
|
||||
ui.write_json(result)
|
||||
|
||||
|
||||
@@ -90,10 +88,8 @@ class CmdPlots(CmdBase):
|
||||
)
|
||||
if html_template_path and not os.path.isabs(html_template_path):
|
||||
assert self.repo.dvc_dir
|
||||
html_template_path = os.path.join(
|
||||
self.repo.dvc_dir, html_template_path
|
||||
)
|
||||
return html_template_path
|
||||
html_template_path = os.path.join(self.repo.dvc_dir, html_template_path)
|
||||
return html_template_path # noqa: RET504
|
||||
|
||||
def run(self) -> int: # noqa: C901
|
||||
from pathlib import Path
|
||||
@@ -106,14 +102,11 @@ class CmdPlots(CmdBase):
|
||||
logger.error("please specify a target for `--show-vega`")
|
||||
return 1
|
||||
if len(self.args.targets) > 1:
|
||||
logger.error(
|
||||
"you can only specify one target for `--show-vega`"
|
||||
)
|
||||
logger.error("you can only specify one target for `--show-vega`")
|
||||
return 1
|
||||
if self.args.json:
|
||||
logger.error(
|
||||
"'--show-vega' and '--json' are mutually exclusive "
|
||||
"options."
|
||||
"'--show-vega' and '--json' are mutually exclusive options."
|
||||
)
|
||||
return 1
|
||||
|
||||
@@ -126,17 +119,14 @@ class CmdPlots(CmdBase):
|
||||
|
||||
if not plots_data:
|
||||
ui.error_write(
|
||||
"No plots were loaded, "
|
||||
"visualization file will not be created."
|
||||
"No plots were loaded, visualization file will not be created."
|
||||
)
|
||||
|
||||
out: str = self.args.out or self.repo.config.get("plots", {}).get(
|
||||
"out_dir", "dvc_plots"
|
||||
)
|
||||
|
||||
renderers_out = (
|
||||
out if self.args.json else os.path.join(out, "static")
|
||||
)
|
||||
renderers_out = out if self.args.json else os.path.join(out, "static")
|
||||
|
||||
renderers = match_defs_renderers(
|
||||
data=plots_data,
|
||||
@@ -249,8 +239,7 @@ def add_parser(subparsers, parent_parser):
|
||||
fix_subparsers(plots_subparsers)
|
||||
|
||||
SHOW_HELP = (
|
||||
"Generate plots from target files or from `plots`"
|
||||
" definitions in `dvc.yaml`."
|
||||
"Generate plots from target files or from `plots` definitions in `dvc.yaml`."
|
||||
)
|
||||
plots_show_parser = plots_subparsers.add_parser(
|
||||
"show",
|
||||
@@ -273,8 +262,7 @@ def add_parser(subparsers, parent_parser):
|
||||
plots_show_parser.set_defaults(func=CmdPlotsShow)
|
||||
|
||||
PLOTS_DIFF_HELP = (
|
||||
"Show multiple versions of a plot by overlaying them "
|
||||
"in a single image."
|
||||
"Show multiple versions of a plot by overlaying them in a single image."
|
||||
)
|
||||
plots_diff_parser = plots_subparsers.add_parser(
|
||||
"diff",
|
||||
@@ -332,9 +320,7 @@ def add_parser(subparsers, parent_parser):
|
||||
)
|
||||
plots_modify_parser.set_defaults(func=CmdPlotsModify)
|
||||
|
||||
TEMPLATES_HELP = (
|
||||
"List built-in plots templates or show JSON specification for one."
|
||||
)
|
||||
TEMPLATES_HELP = "List built-in plots templates or show JSON specification for one."
|
||||
plots_templates_parser = plots_subparsers.add_parser(
|
||||
"templates",
|
||||
parents=[parent_parser],
|
||||
@@ -360,11 +346,8 @@ def _add_props_arguments(parser):
|
||||
"--template",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help=(
|
||||
"Special JSON or HTML schema file to inject with the data. "
|
||||
"See {}".format(
|
||||
help="Special JSON or HTML schema file to inject with the data. See {}".format(
|
||||
format_link("https://man.dvc.org/plots#plot-templates")
|
||||
)
|
||||
),
|
||||
metavar="<path>",
|
||||
).complete = completion.FILE
|
||||
@@ -381,9 +364,7 @@ def _add_props_arguments(parser):
|
||||
default=None, # Use default None to distinguish when it's not used
|
||||
help="Provided CSV or TSV datafile does not have a header.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--title", default=None, metavar="<text>", help="Plot title."
|
||||
)
|
||||
parser.add_argument("--title", default=None, metavar="<text>", help="Plot title.")
|
||||
parser.add_argument(
|
||||
"--x-label", default=None, help="X axis label", metavar="<text>"
|
||||
)
|
||||
|
||||
@@ -20,8 +20,7 @@ class CmdQueueKill(CmdBase):
|
||||
|
||||
def add_parser(queue_subparsers, parent_parser):
|
||||
QUEUE_KILL_HELP = (
|
||||
"Gracefully interrupt running experiment queue tasks "
|
||||
"(equivalent to Ctrl-C)"
|
||||
"Gracefully interrupt running experiment queue tasks (equivalent to Ctrl-C)"
|
||||
)
|
||||
queue_kill_parser = queue_subparsers.add_parser(
|
||||
"kill",
|
||||
|
||||
@@ -34,9 +34,7 @@ def add_parser(queue_subparsers, parent_parser):
|
||||
queue_logs_parser.add_argument(
|
||||
"-e",
|
||||
"--encoding",
|
||||
help=(
|
||||
"Text encoding for log output. Defaults to system locale encoding."
|
||||
),
|
||||
help="Text encoding for log output. Defaults to system locale encoding.",
|
||||
metavar="<encoding>",
|
||||
)
|
||||
queue_logs_parser.add_argument(
|
||||
|
||||
@@ -28,7 +28,6 @@ class CmdQueueRemove(CmdBase):
|
||||
)
|
||||
|
||||
def run(self):
|
||||
|
||||
self.check_arguments()
|
||||
|
||||
if self.args.all:
|
||||
@@ -57,7 +56,6 @@ class CmdQueueRemove(CmdBase):
|
||||
|
||||
|
||||
def add_parser(queue_subparsers, parent_parser):
|
||||
|
||||
QUEUE_REMOVE_HELP = "Remove queued and completed tasks from the queue."
|
||||
queue_remove_parser = queue_subparsers.add_parser(
|
||||
"remove",
|
||||
|
||||
@@ -12,20 +12,15 @@ class CmdQueueStart(CmdBase):
|
||||
"""Start exp queue workers."""
|
||||
|
||||
def run(self):
|
||||
started = self.repo.experiments.celery_queue.start_workers(
|
||||
self.args.jobs
|
||||
)
|
||||
started = self.repo.experiments.celery_queue.start_workers(self.args.jobs)
|
||||
|
||||
suffix = "s" if started > 1 else ""
|
||||
ui.write(
|
||||
f"Started '{started}' new experiments task queue worker{suffix}."
|
||||
)
|
||||
ui.write(f"Started '{started}' new experiments task queue worker{suffix}.")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def add_parser(queue_subparsers, parent_parser):
|
||||
|
||||
QUEUE_START_HELP = "Start the experiments task queue worker."
|
||||
queue_start_parser = queue_subparsers.add_parser(
|
||||
"start",
|
||||
@@ -39,10 +34,7 @@ def add_parser(queue_subparsers, parent_parser):
|
||||
"--jobs",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Maximum number of concurrent queue workers to start. "
|
||||
"Defaults to 1."
|
||||
),
|
||||
help="Maximum number of concurrent queue workers to start. Defaults to 1.",
|
||||
metavar="<number>",
|
||||
)
|
||||
queue_start_parser.set_defaults(func=CmdQueueStart)
|
||||
|
||||
@@ -39,9 +39,7 @@ class CmdQueueStatus(CmdBase):
|
||||
ui.write()
|
||||
|
||||
worker_status = self.repo.experiments.celery_queue.worker_status()
|
||||
active_count = len(
|
||||
[name for name, task in worker_status.items() if task]
|
||||
)
|
||||
active_count = len([name for name, task in worker_status.items() if task])
|
||||
idle_count = len(worker_status) - active_count
|
||||
|
||||
ui.write(f"Worker status: {active_count} active, {idle_count} idle")
|
||||
@@ -50,9 +48,7 @@ class CmdQueueStatus(CmdBase):
|
||||
|
||||
|
||||
def add_parser(queue_subparsers, parent_parser):
|
||||
QUEUE_STATUS_HELP = (
|
||||
"Show the status of experiments queue tasks and workers."
|
||||
)
|
||||
QUEUE_STATUS_HELP = "Show the status of experiments queue tasks and workers."
|
||||
queue_status_parser = queue_subparsers.add_parser(
|
||||
"status",
|
||||
parents=[parent_parser],
|
||||
|
||||
@@ -26,7 +26,6 @@ class CmdQueueStop(CmdBase):
|
||||
|
||||
|
||||
def add_parser(queue_subparsers, parent_parser):
|
||||
|
||||
QUEUE_STOP_HELP = "Stop all experiments task queue workers."
|
||||
queue_stop_parser = queue_subparsers.add_parser(
|
||||
"stop",
|
||||
|
||||
+4
-12
@@ -95,9 +95,7 @@ class CmdRemoteDefault(CmdRemote):
|
||||
if self.args.unset:
|
||||
conf["core"].pop("remote", None)
|
||||
else:
|
||||
merged_conf = self.config.load_config_to_level(
|
||||
self.args.level
|
||||
)
|
||||
merged_conf = self.config.load_config_to_level(self.args.level)
|
||||
if (
|
||||
self.args.name in conf["remote"]
|
||||
or self.args.name in merged_conf["remote"]
|
||||
@@ -208,9 +206,7 @@ def add_parser(subparsers, parent_parser):
|
||||
help=REMOTE_DEFAULT_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_default_parser.add_argument(
|
||||
"name", nargs="?", help="Name of the remote"
|
||||
)
|
||||
remote_default_parser.add_argument("name", nargs="?", help="Name of the remote")
|
||||
remote_default_parser.add_argument(
|
||||
"-u",
|
||||
"--unset",
|
||||
@@ -229,9 +225,7 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_modify_parser.add_argument("name", help="Name of the remote")
|
||||
remote_modify_parser.add_argument(
|
||||
"option", help="Name of the option to modify."
|
||||
)
|
||||
remote_modify_parser.add_argument("option", help="Name of the option to modify.")
|
||||
remote_modify_parser.add_argument(
|
||||
"value", nargs="?", help="(optional) Value of the option."
|
||||
)
|
||||
@@ -262,9 +256,7 @@ def add_parser(subparsers, parent_parser):
|
||||
help=REMOTE_REMOVE_HELP,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
remote_remove_parser.add_argument(
|
||||
"name", help="Name of the remote to remove."
|
||||
)
|
||||
remote_remove_parser.add_argument("name", help="Name of the remote to remove.")
|
||||
remote_remove_parser.set_defaults(func=CmdRemoteRemove)
|
||||
REMOTE_RENAME_HELP = "Rename a DVC remote"
|
||||
remote_rename_parser = remote_subparsers.add_parser(
|
||||
|
||||
@@ -22,8 +22,7 @@ class CmdRemove(CmdBase):
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
REMOVE_HELP = (
|
||||
"Remove stages from dvc.yaml and/or"
|
||||
" stop tracking files or directories."
|
||||
"Remove stages from dvc.yaml and/or stop tracking files or directories."
|
||||
)
|
||||
remove_parser = subparsers.add_parser(
|
||||
"remove",
|
||||
|
||||
+13
-18
@@ -10,9 +10,7 @@ class CmdRepro(CmdBase):
|
||||
def run(self):
|
||||
from dvc.ui import ui
|
||||
|
||||
stages = self.repo.reproduce(
|
||||
**self._common_kwargs, **self._repro_kwargs
|
||||
)
|
||||
stages = self.repo.reproduce(**self._common_kwargs, **self._repro_kwargs)
|
||||
if len(stages) == 0:
|
||||
ui.write(CmdDataStatus.UP_TO_DATE_MSG)
|
||||
else:
|
||||
@@ -55,16 +53,14 @@ def add_arguments(repro_parser):
|
||||
repro_parser.add_argument(
|
||||
"targets",
|
||||
nargs="*",
|
||||
help=(
|
||||
"""\
|
||||
help="""\
|
||||
Stages to reproduce. 'dvc.yaml' by default.
|
||||
The targets can be path to a dvc.yaml file or `.dvc` file,
|
||||
or a stage name from dvc.yaml file from
|
||||
current working directory. To run a stage from dvc.yaml
|
||||
from other directories, the target must be a path followed by colon `:`
|
||||
and then the stage name name.
|
||||
"""
|
||||
),
|
||||
""",
|
||||
).complete = completion.DVCFILES_AND_STAGE
|
||||
repro_parser.add_argument(
|
||||
"-f",
|
||||
@@ -85,16 +81,14 @@ and then the stage name name.
|
||||
"--single-item",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Reproduce only single data item without recursive dependencies "
|
||||
"check.",
|
||||
help="Reproduce only single data item without recursive dependencies check.",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-p",
|
||||
"--pipeline",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Reproduce the whole pipeline that the specified targets "
|
||||
"belong to.",
|
||||
help="Reproduce the whole pipeline that the specified targets belong to.",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-P",
|
||||
@@ -127,8 +121,10 @@ and then the stage name name.
|
||||
"--force-downstream",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Reproduce all descendants of a changed stage even if their "
|
||||
"direct dependencies didn't change.",
|
||||
help=(
|
||||
"Reproduce all descendants of a changed stage even if their "
|
||||
"direct dependencies didn't change."
|
||||
),
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"--pull",
|
||||
@@ -143,15 +139,14 @@ and then the stage name name.
|
||||
"--dry",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Only print the commands that would be executed without "
|
||||
"actually executing.",
|
||||
help=(
|
||||
"Only print the commands that would be executed without actually executing."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
REPRO_HELP = (
|
||||
"Reproduce complete or partial pipelines by executing their stages."
|
||||
)
|
||||
REPRO_HELP = "Reproduce complete or partial pipelines by executing their stages."
|
||||
repro_parser = subparsers.add_parser(
|
||||
"repro",
|
||||
parents=[parent_parser],
|
||||
|
||||
+3
-7
@@ -39,7 +39,7 @@ class CmdRun(CmdBase):
|
||||
{
|
||||
"cmd": parse_cmd(self.args.command),
|
||||
"fname": kwargs.pop("file"),
|
||||
"no_exec": (self.args.no_exec or bool(self.args.checkpoints)),
|
||||
"no_exec": self.args.no_exec or bool(self.args.checkpoints),
|
||||
"run_cache": not kwargs.pop("no_run_cache"),
|
||||
}
|
||||
)
|
||||
@@ -55,9 +55,7 @@ class CmdRun(CmdBase):
|
||||
def add_parser(subparsers, parent_parser):
|
||||
from dvc.commands.stage import _add_common_args
|
||||
|
||||
RUN_HELP = (
|
||||
"Generate a dvc.yaml file from a command and execute the command."
|
||||
)
|
||||
RUN_HELP = "Generate a dvc.yaml file from a command and execute the command."
|
||||
run_parser = subparsers.add_parser(
|
||||
"run",
|
||||
parents=[parent_parser],
|
||||
@@ -66,9 +64,7 @@ def add_parser(subparsers, parent_parser):
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
run_parser.add_argument("-n", "--name", help="Stage name.")
|
||||
run_parser.add_argument(
|
||||
"--file", metavar="<filename>", help=argparse.SUPPRESS
|
||||
)
|
||||
run_parser.add_argument("--file", metavar="<filename>", help=argparse.SUPPRESS)
|
||||
run_parser.add_argument(
|
||||
"--single-stage",
|
||||
action="store_true",
|
||||
|
||||
+15
-17
@@ -44,9 +44,7 @@ def generate_description(stage: "Stage") -> str:
|
||||
return "; ".join(desc)
|
||||
|
||||
|
||||
def prepare_description(
|
||||
stage: "Stage", max_length: int = MAX_TEXT_LENGTH
|
||||
) -> str:
|
||||
def prepare_description(stage: "Stage", max_length: int = MAX_TEXT_LENGTH) -> str:
|
||||
desc = stage.short_description() or generate_description(stage)
|
||||
return truncate_text(desc, max_length)
|
||||
|
||||
@@ -57,9 +55,9 @@ def prepare_stages_data(
|
||||
max_length: int = MAX_TEXT_LENGTH,
|
||||
) -> Dict[str, str]:
|
||||
return {
|
||||
stage.addressing: prepare_description(stage, max_length=max_length)
|
||||
if description
|
||||
else ""
|
||||
stage.addressing: (
|
||||
prepare_description(stage, max_length=max_length) if description else ""
|
||||
)
|
||||
for stage in stages
|
||||
}
|
||||
|
||||
@@ -75,9 +73,7 @@ class CmdStageList(CmdBase):
|
||||
|
||||
# removing duplicates while maintaining order
|
||||
collected = chain.from_iterable(
|
||||
self.repo.stage.collect(
|
||||
target=target, recursive=self.args.recursive
|
||||
)
|
||||
self.repo.stage.collect(target=target, recursive=self.args.recursive)
|
||||
for target in self.args.targets
|
||||
)
|
||||
return dict.fromkeys(collected).keys()
|
||||
@@ -173,8 +169,7 @@ def _add_common_args(parser):
|
||||
"--outs-no-cache",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output file or directory "
|
||||
"(do not put into DVC cache).",
|
||||
help="Declare output file or directory (do not put into DVC cache).",
|
||||
metavar="<filename>",
|
||||
).complete = completion.FILE
|
||||
parser.add_argument(
|
||||
@@ -182,8 +177,10 @@ def _add_common_args(parser):
|
||||
"--checkpoints",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare checkpoint output file or directory for 'dvc exp run'. "
|
||||
"Not compatible with 'dvc repro'.",
|
||||
help=(
|
||||
"Declare checkpoint output file or directory for 'dvc exp run'. "
|
||||
"Not compatible with 'dvc repro'."
|
||||
),
|
||||
metavar="<filename>",
|
||||
).complete = completion.FILE
|
||||
parser.add_argument(
|
||||
@@ -196,16 +193,17 @@ def _add_common_args(parser):
|
||||
"--outs-persist",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output file or directory that will not be "
|
||||
"removed upon repro.",
|
||||
help="Declare output file or directory that will not be removed upon repro.",
|
||||
metavar="<filename>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--outs-persist-no-cache",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Declare output file or directory that will not be "
|
||||
"removed upon repro (do not put into DVC cache).",
|
||||
help=(
|
||||
"Declare output file or directory that will not be "
|
||||
"removed upon repro (do not put into DVC cache)."
|
||||
),
|
||||
metavar="<filename>",
|
||||
)
|
||||
parser.add_argument(
|
||||
|
||||
@@ -81,9 +81,7 @@ class CmdDataStatus(CmdDataBase):
|
||||
if not self.repo.index.stages:
|
||||
ui.write(self.EMPTY_PROJECT_MSG)
|
||||
elif self.args.cloud or self.args.remote:
|
||||
remote = self.args.remote or self.repo.config["core"].get(
|
||||
"remote"
|
||||
)
|
||||
remote = self.args.remote or self.repo.config["core"].get("remote")
|
||||
ui.write(self.IN_SYNC_MSG.format(remote=remote))
|
||||
else:
|
||||
ui.write(self.UP_TO_DATE_MSG)
|
||||
|
||||
@@ -61,8 +61,10 @@ def add_parser(subparsers, parent_parser):
|
||||
"--no-download",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Update .dvc file git revision/hash value(s)"
|
||||
" but do not download the file(s).",
|
||||
help=(
|
||||
"Update .dvc file git revision/hash value(s)"
|
||||
" but do not download the file(s)."
|
||||
),
|
||||
)
|
||||
update_parser.add_argument(
|
||||
"--to-remote",
|
||||
|
||||
@@ -21,9 +21,7 @@ class CmdVersion(CmdBaseNoRepo):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
VERSION_HELP = (
|
||||
"Display the DVC version and system/environment information."
|
||||
)
|
||||
VERSION_HELP = "Display the DVC version and system/environment information."
|
||||
version_parser = subparsers.add_parser(
|
||||
"version",
|
||||
parents=[parent_parser],
|
||||
|
||||
+9
-26
@@ -64,9 +64,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
def keys(self) -> List[str]:
|
||||
return self._keys
|
||||
|
||||
def _iter_col_row(
|
||||
self, row: Sequence["CellT"]
|
||||
) -> Iterator[Tuple["CellT", Column]]:
|
||||
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[Tuple["CellT", Column]]:
|
||||
for val, col in zip_longest(row, self.columns):
|
||||
if col is None:
|
||||
break
|
||||
@@ -103,9 +101,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
...
|
||||
|
||||
@overload
|
||||
def __setitem__(
|
||||
self, item: slice, value: Iterable[Sequence["CellT"]]
|
||||
) -> None:
|
||||
def __setitem__(self, item: slice, value: Iterable[Sequence["CellT"]]) -> None:
|
||||
...
|
||||
|
||||
def __setitem__(self, item, value) -> None:
|
||||
@@ -113,8 +109,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
if isinstance(item, slice):
|
||||
n = len(self.columns)
|
||||
normalized_rows = (
|
||||
chain(val, repeat(self._fill_value, n - len(val)))
|
||||
for val in value
|
||||
chain(val, repeat(self._fill_value, n - len(val))) for val in value
|
||||
)
|
||||
# we need to transpose those rows into columnar format
|
||||
# as we work in terms of column-based arrays
|
||||
@@ -212,9 +207,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
self, cols: Optional[Iterable[str]] = None
|
||||
) -> Iterable[Dict[str, "CellT"]]:
|
||||
keys = self.keys() if cols is None else set(cols)
|
||||
return [
|
||||
{k: self._columns[k][i] for k in keys} for i in range(len(self))
|
||||
]
|
||||
return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
|
||||
|
||||
def dropna( # noqa: C901
|
||||
self,
|
||||
@@ -224,13 +217,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
):
|
||||
if axis not in ["rows", "cols"]:
|
||||
raise ValueError(
|
||||
f"Invalid 'axis' value {axis}."
|
||||
"Choose one of ['rows', 'cols']"
|
||||
f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']"
|
||||
)
|
||||
if how not in ["any", "all"]:
|
||||
raise ValueError(
|
||||
f"Invalid 'how' value {how}. Choose one of ['any', 'all']"
|
||||
)
|
||||
raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']")
|
||||
|
||||
match_line: Set = set()
|
||||
match_any = True
|
||||
@@ -258,11 +248,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
if axis == "rows":
|
||||
for name in self.keys(): # noqa: SIM118
|
||||
self._columns[name] = Column(
|
||||
[
|
||||
x
|
||||
for n, x in enumerate(self._columns[name])
|
||||
if n not in to_drop
|
||||
]
|
||||
[x for n, x in enumerate(self._columns[name]) if n not in to_drop]
|
||||
)
|
||||
else:
|
||||
self.drop(*to_drop)
|
||||
@@ -275,8 +261,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
):
|
||||
if axis not in ["rows", "cols"]:
|
||||
raise ValueError(
|
||||
f"Invalid 'axis' value {axis}."
|
||||
"Choose one of ['rows', 'cols']"
|
||||
f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']"
|
||||
)
|
||||
|
||||
if axis == "cols":
|
||||
@@ -362,9 +347,7 @@ def diff_table(
|
||||
for item, change in sorted(diff_in_file.items()):
|
||||
old_value = with_value(change.get("old"), fill_value)
|
||||
new_value = with_value(change.get("new"), fill_value)
|
||||
diff_value = with_value(
|
||||
change.get("diff", on_empty_diff), fill_value
|
||||
)
|
||||
diff_value = with_value(change.get("diff", on_empty_diff), fill_value)
|
||||
td.append(
|
||||
[
|
||||
fname,
|
||||
|
||||
+2
-5
@@ -153,9 +153,7 @@ class Config(dict):
|
||||
with open(config_file, "w+", encoding="utf-8"):
|
||||
return Config(dvc_dir)
|
||||
|
||||
def load(
|
||||
self, validate: bool = True, config: Optional["DictStrAny"] = None
|
||||
):
|
||||
def load(self, validate: bool = True, config: Optional["DictStrAny"] = None):
|
||||
"""Loads config from all the config files.
|
||||
|
||||
Raises:
|
||||
@@ -376,6 +374,5 @@ def merge(into, update):
|
||||
|
||||
def _lower_keys(data):
|
||||
return {
|
||||
k.lower(): _lower_keys(v) if isinstance(v, dict) else v
|
||||
for k, v in data.items()
|
||||
k.lower(): _lower_keys(v) if isinstance(v, dict) else v for k, v in data.items()
|
||||
}
|
||||
|
||||
+3
-16
@@ -2,16 +2,7 @@ import os
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from funcy import walk_values
|
||||
from voluptuous import (
|
||||
All,
|
||||
Any,
|
||||
Coerce,
|
||||
Invalid,
|
||||
Lower,
|
||||
Optional,
|
||||
Range,
|
||||
Schema,
|
||||
)
|
||||
from voluptuous import All, Any, Coerce, Invalid, Lower, Optional, Range, Schema
|
||||
|
||||
Bool = All(
|
||||
Lower,
|
||||
@@ -34,9 +25,7 @@ def supported_cache_type(types):
|
||||
|
||||
unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
|
||||
if unsupported:
|
||||
raise Invalid(
|
||||
"Unsupported cache type(s): {}".format(", ".join(unsupported))
|
||||
)
|
||||
raise Invalid("Unsupported cache type(s): {}".format(", ".join(unsupported)))
|
||||
|
||||
return types
|
||||
|
||||
@@ -258,9 +247,7 @@ SCHEMA = {
|
||||
"machine": {
|
||||
str: {
|
||||
"cloud": All(Lower, Choices("aws", "azure")),
|
||||
"region": All(
|
||||
Lower, Choices("us-west", "us-east", "eu-west", "eu-north")
|
||||
),
|
||||
"region": All(Lower, Choices("us-west", "us-east", "eu-west", "eu-north")),
|
||||
"image": str,
|
||||
"spot": Bool,
|
||||
"spot_price": Coerce(float),
|
||||
|
||||
+1
-3
@@ -40,9 +40,7 @@ def _spawn_windows(cmd, env):
|
||||
startupinfo = STARTUPINFO()
|
||||
startupinfo.dwFlags |= STARTF_USESHOWWINDOW
|
||||
|
||||
_popen(
|
||||
cmd, env=env, creationflags=creationflags, startupinfo=startupinfo
|
||||
)
|
||||
_popen(cmd, env=env, creationflags=creationflags, startupinfo=startupinfo)
|
||||
|
||||
|
||||
def _spawn_posix(cmd, env):
|
||||
|
||||
+1
-3
@@ -272,8 +272,6 @@ def draw(vertices, edges):
|
||||
vertex.view.h,
|
||||
)
|
||||
|
||||
canvas.text(
|
||||
int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
|
||||
)
|
||||
canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
|
||||
|
||||
return canvas.draw()
|
||||
|
||||
+3
-1
@@ -111,8 +111,10 @@ class DataCloud:
|
||||
for hash_info in status.missing
|
||||
)
|
||||
logger.warning(
|
||||
(
|
||||
"Some of the cache files do not exist neither locally "
|
||||
"nor on remote. Missing cache files:\n%s",
|
||||
"nor on remote. Missing cache files:\n%s"
|
||||
),
|
||||
missing_desc,
|
||||
)
|
||||
|
||||
|
||||
@@ -90,6 +90,4 @@ def _merge_params(s_list):
|
||||
|
||||
def loads_params(stage, s_list):
|
||||
d = _merge_params(s_list)
|
||||
return [
|
||||
ParamsDependency(stage, path, params) for path, params in d.items()
|
||||
]
|
||||
return [ParamsDependency(stage, path, params) for path, params in d.items()]
|
||||
|
||||
@@ -47,9 +47,7 @@ class Dependency(Output):
|
||||
self.def_path = self.fs.path.version_path(
|
||||
self.def_path, self.meta.version_id
|
||||
)
|
||||
self.fs_path = self.fs.path.version_path(
|
||||
self.fs_path, self.meta.version_id
|
||||
)
|
||||
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
|
||||
|
||||
def download(self, to, jobs=None):
|
||||
fs_download(self.fs, self.fs_path, to, jobs=jobs)
|
||||
@@ -57,9 +55,7 @@ class Dependency(Output):
|
||||
def save(self):
|
||||
super().save()
|
||||
if self.fs.version_aware:
|
||||
self.fs_path = self.fs.path.version_path(
|
||||
self.fs_path, self.meta.version_id
|
||||
)
|
||||
self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id)
|
||||
|
||||
def dumpd(self, **kwargs):
|
||||
if self.fs.version_aware:
|
||||
|
||||
@@ -154,9 +154,7 @@ class ParamsDependency(Dependency):
|
||||
try:
|
||||
return load_path(self.fs_path, self.repo.fs)
|
||||
except ParseError as exc:
|
||||
raise BadParamFileError(
|
||||
f"Unable to read parameters from '{self}'"
|
||||
) from exc
|
||||
raise BadParamFileError(f"Unable to read parameters from '{self}'") from exc
|
||||
|
||||
def get_hash(self):
|
||||
info = self.read_params()
|
||||
|
||||
+7
-22
@@ -35,9 +35,7 @@ class RepoDependency(Dependency):
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs
|
||||
):
|
||||
def __init__(self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs):
|
||||
self.def_repo = def_repo
|
||||
self._objs: Dict[str, "HashFile"] = {}
|
||||
self._meta: Dict[str, "Meta"] = {}
|
||||
@@ -100,17 +98,13 @@ class RepoDependency(Dependency):
|
||||
# immutable, hence its impossible for checksum to change.
|
||||
return False
|
||||
|
||||
def get_used_objs(
|
||||
self, **kwargs
|
||||
) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
|
||||
def get_used_objs(self, **kwargs) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
|
||||
used, _, _ = self._get_used_and_obj(**kwargs)
|
||||
return used
|
||||
|
||||
def _get_used_and_obj(
|
||||
self, obj_only: bool = False, **kwargs
|
||||
) -> Tuple[
|
||||
Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"
|
||||
]:
|
||||
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"]:
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.exceptions import NoOutputOrStageError, PathMissingError
|
||||
from dvc.utils import as_posix
|
||||
@@ -163,9 +157,7 @@ class RepoDependency(Dependency):
|
||||
used_obj_ids[object_store].update(oid for _, _, oid in obj)
|
||||
return used_obj_ids, meta, obj
|
||||
|
||||
def _check_circular_import(
|
||||
self, odb: "ObjectDB", obj_ids: Set["HashInfo"]
|
||||
) -> None:
|
||||
def _check_circular_import(self, odb: "ObjectDB", obj_ids: Set["HashInfo"]) -> None:
|
||||
from dvc.exceptions import CircularImportError
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
from dvc_data.hashfile.db.reference import ReferenceHashFileDB
|
||||
@@ -187,10 +179,7 @@ class RepoDependency(Dependency):
|
||||
for obj in iter_objs():
|
||||
if not isinstance(obj.fs, DVCFileSystem):
|
||||
continue
|
||||
if (
|
||||
obj.fs.repo_url in checked_urls
|
||||
or obj.fs.repo.root_dir in checked_urls
|
||||
):
|
||||
if obj.fs.repo_url in checked_urls or obj.fs.repo.root_dir in checked_urls:
|
||||
continue
|
||||
self_url = self.repo.url or self.repo.root_dir
|
||||
if (
|
||||
@@ -221,9 +210,7 @@ class RepoDependency(Dependency):
|
||||
)
|
||||
return meta
|
||||
|
||||
def _make_repo(
|
||||
self, locked: bool = True, **kwargs
|
||||
) -> "ContextManager[Repo]":
|
||||
def _make_repo(self, locked: bool = True, **kwargs) -> "ContextManager[Repo]":
|
||||
from dvc.external_repo import external_repo
|
||||
|
||||
d = self.def_repo
|
||||
@@ -232,6 +219,4 @@ class RepoDependency(Dependency):
|
||||
|
||||
def _get_rev(self, locked: bool = True):
|
||||
d = self.def_repo
|
||||
return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(
|
||||
self.PARAM_REV
|
||||
)
|
||||
return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(self.PARAM_REV)
|
||||
|
||||
+8
-25
@@ -63,9 +63,7 @@ def is_valid_filename(path):
|
||||
|
||||
|
||||
def is_dvc_file(path):
|
||||
return os.path.isfile(path) and (
|
||||
is_valid_filename(path) or is_lock_file(path)
|
||||
)
|
||||
return os.path.isfile(path) and (is_valid_filename(path) or is_lock_file(path))
|
||||
|
||||
|
||||
def is_lock_file(path):
|
||||
@@ -77,9 +75,7 @@ def is_git_ignored(repo, path):
|
||||
from dvc.scm import NoSCMError
|
||||
|
||||
try:
|
||||
return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(
|
||||
path
|
||||
)
|
||||
return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(path)
|
||||
except NoSCMError:
|
||||
return False
|
||||
|
||||
@@ -152,9 +148,7 @@ class FileMixin:
|
||||
# 4. when the file is git ignored
|
||||
if not self.exists():
|
||||
dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path)
|
||||
raise StageFileDoesNotExistError(
|
||||
self.path, dvc_ignored=dvc_ignored
|
||||
)
|
||||
raise StageFileDoesNotExistError(self.path, dvc_ignored=dvc_ignored)
|
||||
|
||||
self._verify_filename()
|
||||
if not self.repo.fs.isfile(self.path):
|
||||
@@ -271,9 +265,7 @@ class ProjectFile(FileMixin):
|
||||
@staticmethod
|
||||
def _check_if_parametrized(stage, action: str = "dump") -> None:
|
||||
if stage.raw_data.parametrized:
|
||||
raise ParametrizedDumpError(
|
||||
f"cannot {action} a parametrized {stage}"
|
||||
)
|
||||
raise ParametrizedDumpError(f"cannot {action} a parametrized {stage}")
|
||||
|
||||
def _dump_pipeline_file(self, stage):
|
||||
self._check_if_parametrized(stage)
|
||||
@@ -286,9 +278,7 @@ class ProjectFile(FileMixin):
|
||||
data["stages"] = data.get("stages", {})
|
||||
existing_entry = stage.name in data["stages"]
|
||||
action = "Modifying" if existing_entry else "Adding"
|
||||
logger.info(
|
||||
"%s stage '%s' in '%s'", action, stage.name, self.relpath
|
||||
)
|
||||
logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
|
||||
|
||||
if existing_entry:
|
||||
orig_stage_data = data["stages"][stage.name]
|
||||
@@ -300,9 +290,7 @@ class ProjectFile(FileMixin):
|
||||
|
||||
@property
|
||||
def stage(self):
|
||||
raise DvcException(
|
||||
"ProjectFile has multiple stages. Please specify it's name."
|
||||
)
|
||||
raise DvcException("ProjectFile has multiple stages. Please specify it's name.")
|
||||
|
||||
@cached_property
|
||||
def contents(self) -> Dict[str, Any]:
|
||||
@@ -366,10 +354,7 @@ class ProjectFile(FileMixin):
|
||||
|
||||
|
||||
def get_lockfile_schema(d):
|
||||
from dvc.schema import (
|
||||
COMPILED_LOCKFILE_V1_SCHEMA,
|
||||
COMPILED_LOCKFILE_V2_SCHEMA,
|
||||
)
|
||||
from dvc.schema import COMPILED_LOCKFILE_V1_SCHEMA, COMPILED_LOCKFILE_V2_SCHEMA
|
||||
|
||||
schema = {
|
||||
LOCKFILE_VERSION.V1: COMPILED_LOCKFILE_V1_SCHEMA,
|
||||
@@ -422,9 +407,7 @@ class Lockfile(FileMixin):
|
||||
with modify_yaml(self.path, fs=self.repo.fs) as data:
|
||||
version = LOCKFILE_VERSION.from_dict(data)
|
||||
if version == LOCKFILE_VERSION.V1:
|
||||
logger.info(
|
||||
"Migrating lock file '%s' from v1 to v2", self.relpath
|
||||
)
|
||||
logger.info("Migrating lock file '%s' from v1 to v2", self.relpath)
|
||||
migrate_lock_v1_to_v2(data, self.latest_version_info)
|
||||
else:
|
||||
if not data:
|
||||
|
||||
+9
-20
@@ -150,8 +150,9 @@ class CyclicGraphError(DvcException):
|
||||
class ConfirmRemoveError(DvcException):
|
||||
def __init__(self, path):
|
||||
super().__init__(
|
||||
"unable to remove '{}' without a confirmation. Use "
|
||||
"`-f` to force.".format(path)
|
||||
"unable to remove '{}' without a confirmation. Use `-f` to force.".format(
|
||||
path
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -177,9 +178,7 @@ class BadMetricError(DvcException):
|
||||
|
||||
class RecursiveAddingWhileUsingFilename(DvcException):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
"cannot use `fname` with multiple targets or `-R|--recursive`"
|
||||
)
|
||||
super().__init__("cannot use `fname` with multiple targets or `-R|--recursive`")
|
||||
|
||||
|
||||
class OverlappingOutputPathsError(DvcException):
|
||||
@@ -206,9 +205,7 @@ class FileMissingError(DvcException):
|
||||
def __init__(self, path, hint=None):
|
||||
self.path = path
|
||||
hint = "" if hint is None else f". {hint}"
|
||||
super().__init__(
|
||||
f"Can't find '{path}' neither locally nor on remote{hint}"
|
||||
)
|
||||
super().__init__(f"Can't find '{path}' neither locally nor on remote{hint}")
|
||||
|
||||
|
||||
class FileTransferError(DvcException):
|
||||
@@ -250,9 +247,7 @@ class CollectCacheError(DvcException):
|
||||
|
||||
class NoRemoteInExternalRepoError(DvcException):
|
||||
def __init__(self, url):
|
||||
super().__init__(
|
||||
f"No DVC remote is specified in target repository '{url}'."
|
||||
)
|
||||
super().__init__(f"No DVC remote is specified in target repository '{url}'.")
|
||||
|
||||
|
||||
class NoOutputInExternalRepoError(DvcException):
|
||||
@@ -277,8 +272,7 @@ class PathMissingError(DvcException):
|
||||
" neither as a DVC output nor as a Git-tracked file."
|
||||
)
|
||||
default_msg_dvc_only = (
|
||||
"The path '{}' does not exist in the target repository '{}'"
|
||||
" as an DVC output."
|
||||
"The path '{}' does not exist in the target repository '{}' as an DVC output."
|
||||
)
|
||||
|
||||
def __init__(self, path, repo, dvc_only=False):
|
||||
@@ -294,7 +288,6 @@ class URLMissingError(DvcException):
|
||||
|
||||
class RemoteCacheRequiredError(DvcException):
|
||||
def __init__(self, scheme, fs_path):
|
||||
|
||||
super().__init__(
|
||||
(
|
||||
"Current operation was unsuccessful because '{}' requires "
|
||||
@@ -319,8 +312,7 @@ class NoOutputOrStageError(DvcException):
|
||||
|
||||
def __init__(self, target, file):
|
||||
super().__init__(
|
||||
f"'{target}' "
|
||||
f"does not exist as an output or a stage name in '{file}'"
|
||||
f"'{target}' does not exist as an output or a stage name in '{file}'"
|
||||
)
|
||||
|
||||
|
||||
@@ -329,11 +321,8 @@ class MergeError(DvcException):
|
||||
|
||||
|
||||
class CacheLinkError(DvcException):
|
||||
|
||||
SUPPORT_LINK = "See {} for more information.".format(
|
||||
format_link(
|
||||
"https://dvc.org/doc/user-guide/troubleshooting#cache-types"
|
||||
)
|
||||
format_link("https://dvc.org/doc/user-guide/troubleshooting#cache-types")
|
||||
)
|
||||
|
||||
def __init__(self, fs_paths):
|
||||
|
||||
@@ -88,9 +88,7 @@ def external_repo(
|
||||
raise NoRemoteInExternalRepoError(url) from exc
|
||||
except OutputNotFoundError as exc:
|
||||
if exc.repo is repo:
|
||||
raise NoOutputInExternalRepoError(
|
||||
exc.output, repo.root_dir, url
|
||||
) from exc
|
||||
raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
|
||||
raise
|
||||
except FileMissingError as exc:
|
||||
raise PathMissingError(exc.path, url) from exc
|
||||
@@ -234,9 +232,7 @@ def _clone_default_branch(url, rev, for_write=False): # noqa: C901
|
||||
os.path.join(clone_path, Git.GIT_DIR, "shallow")
|
||||
)
|
||||
if shallow:
|
||||
logger.debug(
|
||||
"erepo: using shallow clone for branch '%s'", rev
|
||||
)
|
||||
logger.debug("erepo: using shallow clone for branch '%s'", rev)
|
||||
except CloneError:
|
||||
git_dir = os.path.join(clone_path, ".git")
|
||||
if os.path.exists(git_dir):
|
||||
@@ -292,8 +288,6 @@ def _remove(path):
|
||||
try:
|
||||
os_retry(remove)(path)
|
||||
except PermissionError:
|
||||
logger.warning(
|
||||
"Failed to remove '%s'", relpath(path), exc_info=True
|
||||
)
|
||||
logger.warning("Failed to remove '%s'", relpath(path), exc_info=True)
|
||||
else:
|
||||
remove(path)
|
||||
|
||||
+1
-3
@@ -72,9 +72,7 @@ def get_fs_config(config, **kwargs):
|
||||
except KeyError:
|
||||
from dvc.config import RemoteNotFoundError
|
||||
|
||||
raise RemoteNotFoundError( # noqa: B904
|
||||
f"remote '{name}' doesn't exist"
|
||||
)
|
||||
raise RemoteNotFoundError(f"remote '{name}' doesn't exist") # noqa: B904
|
||||
else:
|
||||
remote_conf = kwargs
|
||||
return _resolve_remote_refs(config, remote_conf)
|
||||
|
||||
+1
-3
@@ -58,9 +58,7 @@ class RichCallback(Callback):
|
||||
|
||||
@cached_property
|
||||
def task(self) -> "TaskID":
|
||||
return self.progress.add_task(
|
||||
**self._task_kwargs # type: ignore[arg-type]
|
||||
)
|
||||
return self.progress.add_task(**self._task_kwargs) # type: ignore[arg-type]
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
+5
-22
@@ -6,16 +6,7 @@ import os
|
||||
import posixpath
|
||||
import threading
|
||||
from contextlib import suppress
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
|
||||
|
||||
from fsspec.spec import AbstractFileSystem
|
||||
from funcy import wrap_with
|
||||
@@ -126,9 +117,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
super().__init__()
|
||||
if repo is None:
|
||||
repo = self._make_repo(
|
||||
url=url, rev=rev, subrepos=subrepos, **repo_kwargs
|
||||
)
|
||||
repo = self._make_repo(url=url, rev=rev, subrepos=subrepos, **repo_kwargs)
|
||||
assert repo is not None
|
||||
# pylint: disable=protected-access
|
||||
repo_factory = repo._fs_conf["repo_factory"]
|
||||
@@ -144,9 +133,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
relparts = ()
|
||||
assert repo is not None
|
||||
if repo.fs.path.isin(repo.fs.path.getcwd(), repo.root_dir):
|
||||
relparts = repo.fs.path.relparts(
|
||||
repo.fs.path.getcwd(), repo.root_dir
|
||||
)
|
||||
relparts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.root_dir)
|
||||
return self.root_marker + self.sep.join(relparts)
|
||||
|
||||
self.path = Path(self.sep, getcwd=_getcwd)
|
||||
@@ -222,9 +209,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
scm=self.repo.scm,
|
||||
repo_factory=self.repo_factory,
|
||||
)
|
||||
self._datafss[key] = DataFileSystem(
|
||||
index=repo.index.data["repo"]
|
||||
)
|
||||
self._datafss[key] = DataFileSystem(index=repo.index.data["repo"])
|
||||
self._subrepos_trie[key] = repo
|
||||
|
||||
def _is_dvc_repo(self, dir_path):
|
||||
@@ -331,9 +316,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
ignore_subrepos = kwargs.get("ignore_subrepos", True)
|
||||
return self._info(key, path, ignore_subrepos=ignore_subrepos)
|
||||
|
||||
def _info( # noqa: C901
|
||||
self, key, path, ignore_subrepos=True, check_ignored=True
|
||||
):
|
||||
def _info(self, key, path, ignore_subrepos=True, check_ignored=True): # noqa: C901
|
||||
repo, dvc_fs, subkey = self._get_subrepo_info(key)
|
||||
|
||||
dvc_info = None
|
||||
|
||||
+10
-26
@@ -30,9 +30,7 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
from pathspec.patterns.gitwildmatch import _DIR_MARK
|
||||
|
||||
if pattern_list and isinstance(pattern_list[0], str):
|
||||
pattern_list = [
|
||||
PatternInfo(pattern, "") for pattern in pattern_list
|
||||
]
|
||||
pattern_list = [PatternInfo(pattern, "") for pattern in pattern_list]
|
||||
|
||||
self.sep = sep
|
||||
self.pattern_list = pattern_list
|
||||
@@ -40,20 +38,14 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
|
||||
self.regex_pattern_list = []
|
||||
for count, pattern in enumerate(pattern_list):
|
||||
pattern, group = GitWildMatchPattern.pattern_to_regex(
|
||||
pattern.patterns
|
||||
)
|
||||
pattern, group = GitWildMatchPattern.pattern_to_regex(pattern.patterns)
|
||||
if pattern:
|
||||
pattern = pattern.replace(
|
||||
f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>"
|
||||
)
|
||||
pattern = pattern.replace(f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>")
|
||||
self.regex_pattern_list.append((pattern, group))
|
||||
|
||||
self.ignore_spec = [
|
||||
(ignore, re.compile("|".join(item[0] for item in group)))
|
||||
for ignore, group in groupby(
|
||||
self.regex_pattern_list, lambda x: x[1]
|
||||
)
|
||||
for ignore, group in groupby(self.regex_pattern_list, lambda x: x[1])
|
||||
if ignore is not None
|
||||
]
|
||||
|
||||
@@ -64,9 +56,7 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
with fs.open(path, encoding="utf-8") as fobj:
|
||||
path_spec_lines = [
|
||||
PatternInfo(line, f"{name}:{line_no + 1}:{line}")
|
||||
for line_no, line in enumerate(
|
||||
map(str.strip, fobj.readlines())
|
||||
)
|
||||
for line_no, line in enumerate(map(str.strip, fobj.readlines()))
|
||||
if line and not (line.strip().startswith("#"))
|
||||
]
|
||||
|
||||
@@ -158,9 +148,7 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
return bool(self.pattern_list)
|
||||
|
||||
|
||||
CheckIgnoreResult = namedtuple(
|
||||
"CheckIgnoreResult", ["file", "match", "patterns"]
|
||||
)
|
||||
CheckIgnoreResult = namedtuple("CheckIgnoreResult", ["file", "match", "patterns"])
|
||||
|
||||
|
||||
def _no_match(path):
|
||||
@@ -249,9 +237,7 @@ class DvcIgnoreFilter:
|
||||
dnames = []
|
||||
|
||||
for dname in dnames:
|
||||
self._update_sub_repo(
|
||||
self.fs.path.join(dirname, dname), ignore_trie
|
||||
)
|
||||
self._update_sub_repo(self.fs.path.join(dirname, dname), ignore_trie)
|
||||
|
||||
def _update_sub_repo(self, path, ignore_trie: Trie):
|
||||
from dvc.repo import Repo
|
||||
@@ -409,9 +395,7 @@ class DvcIgnoreFilter:
|
||||
# https://github.com/iterative/dvc/issues/5046
|
||||
full_target = self.fs.path.abspath(target)
|
||||
if not self._outside_repo(full_target):
|
||||
dirname, basename = self.fs.path.split(
|
||||
self.fs.path.normpath(full_target)
|
||||
)
|
||||
dirname, basename = self.fs.path.split(self.fs.path.normpath(full_target))
|
||||
pattern = self._get_trie_pattern(dirname)
|
||||
if pattern:
|
||||
matches = pattern.matches(
|
||||
@@ -433,9 +417,9 @@ class DvcIgnoreFilter:
|
||||
return self.is_ignored_file(path, ignore_subrepos)
|
||||
if fs.isdir(path):
|
||||
return self.is_ignored_dir(path, ignore_subrepos)
|
||||
return self.is_ignored_file(
|
||||
return self.is_ignored_file(path, ignore_subrepos) or self.is_ignored_dir(
|
||||
path, ignore_subrepos
|
||||
) or self.is_ignored_dir(path, ignore_subrepos)
|
||||
)
|
||||
|
||||
|
||||
def init(path):
|
||||
|
||||
+2
-4
@@ -30,8 +30,7 @@ def get_dvc_info():
|
||||
info = [
|
||||
dvc_version,
|
||||
"-" * len(dvc_version),
|
||||
f"Platform: Python {platform.python_version()} on "
|
||||
f"{platform.platform()}",
|
||||
f"Platform: Python {platform.python_version()} on {platform.platform()}",
|
||||
f"Subprojects:{_get_subprojects()}",
|
||||
f"Supports:{_get_supported_remotes()}",
|
||||
]
|
||||
@@ -125,8 +124,7 @@ def _get_supported_remotes():
|
||||
dependencies = []
|
||||
for requirement in fs_cls.REQUIRES:
|
||||
dependencies.append(
|
||||
f"{requirement} = "
|
||||
f"{importlib_metadata.version(requirement)}"
|
||||
f"{requirement} = {importlib_metadata.version(requirement)}"
|
||||
)
|
||||
|
||||
remote_info = scheme
|
||||
|
||||
+3
-9
@@ -20,9 +20,7 @@ FAILED_TO_LOCK_MESSAGE = (
|
||||
"Unable to acquire lock. Most likely another DVC process is running or "
|
||||
"was terminated abruptly. Check the page {} for other possible reasons "
|
||||
"and to learn how to resolve this."
|
||||
).format(
|
||||
format_link("https://dvc.org/doc/user-guide/troubleshooting#lock-issue")
|
||||
)
|
||||
).format(format_link("https://dvc.org/doc/user-guide/troubleshooting#lock-issue"))
|
||||
|
||||
|
||||
class LockError(DvcException):
|
||||
@@ -61,9 +59,7 @@ class LockBase(ABC):
|
||||
|
||||
|
||||
class LockNoop(LockBase):
|
||||
def __init__(
|
||||
self, *args, **kwargs
|
||||
): # pylint: disable=super-init-not-called
|
||||
def __init__(self, *args, **kwargs): # pylint: disable=super-init-not-called
|
||||
self._lock = False
|
||||
|
||||
def lock(self):
|
||||
@@ -107,10 +103,8 @@ class Lock(LockBase):
|
||||
with Tqdm(
|
||||
bar_format="{desc}",
|
||||
disable=not self._friendly,
|
||||
desc=(
|
||||
"If DVC froze, see `hardlink_lock` in {}".format(
|
||||
desc="If DVC froze, see `hardlink_lock` in {}".format(
|
||||
format_link("https://man.dvc.org/config#core")
|
||||
)
|
||||
),
|
||||
):
|
||||
self._lock = zc.lockfile.LockFile(self._lockfile)
|
||||
|
||||
+3
-12
@@ -159,9 +159,7 @@ class LoggerHandler(logging.StreamHandler):
|
||||
pass # noqa: S110
|
||||
|
||||
msg = self.format(record)
|
||||
Tqdm.write(
|
||||
msg, file=self.stream, end=getattr(self, "terminator", "\n")
|
||||
)
|
||||
Tqdm.write(msg, file=self.stream, end=getattr(self, "terminator", "\n"))
|
||||
self.flush()
|
||||
except (BrokenPipeError, RecursionError):
|
||||
raise
|
||||
@@ -171,9 +169,7 @@ class LoggerHandler(logging.StreamHandler):
|
||||
|
||||
def _is_verbose():
|
||||
return (
|
||||
logging.NOTSET
|
||||
< logging.getLogger("dvc").getEffectiveLevel()
|
||||
<= logging.DEBUG
|
||||
logging.NOTSET < logging.getLogger("dvc").getEffectiveLevel() <= logging.DEBUG
|
||||
)
|
||||
|
||||
|
||||
@@ -186,17 +182,12 @@ def _iter_causes(exc):
|
||||
def _stack_trace(exc_info):
|
||||
import traceback
|
||||
|
||||
return (
|
||||
"\n"
|
||||
"{red}{line}{nc}\n"
|
||||
"{trace}"
|
||||
"{red}{line}{nc}".format(
|
||||
return "\n{red}{line}{nc}\n{trace}{red}{line}{nc}".format(
|
||||
red=colorama.Fore.RED,
|
||||
line="-" * 60,
|
||||
trace="".join(traceback.format_exception(*exc_info)),
|
||||
nc=colorama.Fore.RESET,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def disable_other_loggers():
|
||||
|
||||
@@ -67,9 +67,7 @@ class MachineBackends(Mapping):
|
||||
initialized = self.initialized.get(key)
|
||||
if not initialized:
|
||||
backend = self.backends[key]
|
||||
initialized = backend(
|
||||
os.path.join(self.tmp_dir, key), **self.kwargs
|
||||
)
|
||||
initialized = backend(os.path.join(self.tmp_dir, key), **self.kwargs)
|
||||
self.initialized[key] = initialized
|
||||
return initialized
|
||||
|
||||
|
||||
@@ -21,9 +21,7 @@ class BaseMachineBackend(ABC):
|
||||
"""Stop and destroy all instances of the specified machine."""
|
||||
|
||||
@abstractmethod
|
||||
def instances(
|
||||
self, name: Optional[str] = None, **config
|
||||
) -> Iterator[dict]:
|
||||
def instances(self, name: Optional[str] = None, **config) -> Iterator[dict]:
|
||||
"""Iterate over status of all instances of the specified machine."""
|
||||
|
||||
def close(self): # noqa: B027
|
||||
|
||||
@@ -56,12 +56,8 @@ class TerraformBackend(BaseMachineBackend):
|
||||
|
||||
create = partialmethod(_tpi_func, "create") # type: ignore[assignment]
|
||||
destroy = partialmethod(_tpi_func, "destroy") # type: ignore[assignment]
|
||||
instances = partialmethod(
|
||||
_tpi_func, "instances"
|
||||
) # type: ignore[assignment]
|
||||
run_shell = partialmethod(
|
||||
_tpi_func, "run_shell"
|
||||
) # type: ignore[assignment]
|
||||
instances = partialmethod(_tpi_func, "instances") # type: ignore[assignment]
|
||||
run_shell = partialmethod(_tpi_func, "run_shell") # type: ignore[assignment]
|
||||
|
||||
def get_executor_kwargs(self, name: str, **config) -> dict:
|
||||
with self.make_tpi(name) as tpi:
|
||||
|
||||
+13
-38
@@ -151,9 +151,7 @@ def _merge_data(s_list):
|
||||
d[key].update({})
|
||||
continue
|
||||
if not isinstance(key, dict):
|
||||
raise ValueError( # noqa: TRY004
|
||||
f"'{type(key).__name__}' not supported."
|
||||
)
|
||||
raise ValueError(f"'{type(key).__name__}' not supported.") # noqa: TRY004
|
||||
|
||||
for k, flags in key.items():
|
||||
if not isinstance(flags, dict):
|
||||
@@ -336,9 +334,7 @@ class Output:
|
||||
if fs_config is not None:
|
||||
fs_kwargs.update(**fs_config)
|
||||
|
||||
fs_cls, fs_config, fs_path = get_cloud_fs(
|
||||
self.repo, url=path, **fs_kwargs
|
||||
)
|
||||
fs_cls, fs_config, fs_path = get_cloud_fs(self.repo, url=path, **fs_kwargs)
|
||||
self.fs = fs_cls(**fs_config)
|
||||
|
||||
if (
|
||||
@@ -500,9 +496,7 @@ class Output:
|
||||
|
||||
@property
|
||||
def cache_path(self):
|
||||
return self.odb.fs.unstrip_protocol(
|
||||
self.odb.oid_to_path(self.hash_info.value)
|
||||
)
|
||||
return self.odb.fs.unstrip_protocol(self.odb.oid_to_path(self.hash_info.value))
|
||||
|
||||
def get_hash(self):
|
||||
_, hash_info = self._get_hash_meta()
|
||||
@@ -691,9 +685,7 @@ class Output:
|
||||
dry_run=True,
|
||||
)
|
||||
if not self.IS_DEPENDENCY:
|
||||
logger.debug(
|
||||
"Output '%s' doesn't use cache. Skipping saving.", self
|
||||
)
|
||||
logger.debug("Output '%s' doesn't use cache. Skipping saving.", self)
|
||||
|
||||
self.hash_info = self.obj.hash_info
|
||||
self.files = None
|
||||
@@ -724,9 +716,7 @@ class Output:
|
||||
|
||||
if self.use_cache:
|
||||
granular = (
|
||||
self.is_dir_checksum
|
||||
and filter_info
|
||||
and filter_info != self.fs_path
|
||||
self.is_dir_checksum and filter_info and filter_info != self.fs_path
|
||||
)
|
||||
if granular:
|
||||
obj = self._commit_granular_dir(filter_info)
|
||||
@@ -757,9 +747,7 @@ class Output:
|
||||
self.set_exec()
|
||||
|
||||
def _commit_granular_dir(self, filter_info) -> Optional["HashFile"]:
|
||||
prefix = self.fs.path.parts(
|
||||
self.fs.path.relpath(filter_info, self.fs_path)
|
||||
)
|
||||
prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
|
||||
staging, _, save_obj = build(
|
||||
self.odb,
|
||||
self.fs_path,
|
||||
@@ -794,9 +782,7 @@ class Output:
|
||||
ret.update(split_file_meta_from_cloud(meta_d))
|
||||
|
||||
if self.is_in_repo:
|
||||
path = self.fs.path.as_posix(
|
||||
relpath(self.fs_path, self.stage.wdir)
|
||||
)
|
||||
path = self.fs.path.as_posix(relpath(self.fs_path, self.stage.wdir))
|
||||
else:
|
||||
path = self.def_path
|
||||
|
||||
@@ -837,16 +823,13 @@ class Output:
|
||||
if obj:
|
||||
obj = cast("Tree", obj)
|
||||
ret[self.PARAM_FILES] = [
|
||||
split_file_meta_from_cloud(f)
|
||||
for f in obj.as_list(with_meta=True)
|
||||
split_file_meta_from_cloud(f) for f in obj.as_list(with_meta=True)
|
||||
]
|
||||
return ret
|
||||
|
||||
def verify_metric(self):
|
||||
if self.fs.protocol != "local":
|
||||
raise DvcException(
|
||||
f"verify metric is not supported for {self.protocol}"
|
||||
)
|
||||
raise DvcException(f"verify metric is not supported for {self.protocol}")
|
||||
if not self.metric:
|
||||
return
|
||||
|
||||
@@ -902,9 +885,7 @@ class Output:
|
||||
) -> Optional[Tuple[bool, Optional[bool]]]:
|
||||
if not self.use_cache:
|
||||
if progress_callback != DEFAULT_CALLBACK:
|
||||
progress_callback.relative_update(
|
||||
self.get_files_number(filter_info)
|
||||
)
|
||||
progress_callback.relative_update(self.get_files_number(filter_info))
|
||||
return None
|
||||
|
||||
obj = self.get_obj(filter_info=filter_info)
|
||||
@@ -1071,9 +1052,7 @@ class Output:
|
||||
obj = self.get_obj()
|
||||
if filter_info and filter_info != self.fs_path:
|
||||
assert obj
|
||||
prefix = self.fs.path.parts(
|
||||
self.fs.path.relpath(filter_info, self.fs_path)
|
||||
)
|
||||
prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
|
||||
obj = cast(Tree, obj)
|
||||
return obj.filter(prefix)
|
||||
return cast(Tree, obj)
|
||||
@@ -1181,14 +1160,10 @@ class Output:
|
||||
other.pop(opt, None)
|
||||
|
||||
if my != other:
|
||||
raise MergeError(
|
||||
"unable to auto-merge outputs with different options"
|
||||
)
|
||||
raise MergeError("unable to auto-merge outputs with different options")
|
||||
|
||||
if not out.is_dir_checksum:
|
||||
raise MergeError(
|
||||
"unable to auto-merge outputs that are not directories"
|
||||
)
|
||||
raise MergeError("unable to auto-merge outputs that are not directories")
|
||||
|
||||
def merge(self, ancestor, other, allowed=None):
|
||||
from dvc_data.hashfile.tree import MergeError as TreeMergeError
|
||||
|
||||
+11
-18
@@ -72,9 +72,7 @@ def _format_preamble(msg: str, path: str, spacing: str = " ") -> str:
|
||||
|
||||
def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn":
|
||||
spacing = (
|
||||
"\n"
|
||||
if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded))
|
||||
else " "
|
||||
"\n" if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded)) else " "
|
||||
)
|
||||
message = _format_preamble(msg, path, spacing) + str(exc)
|
||||
|
||||
@@ -116,8 +114,7 @@ def check_interpolations(data: "DictStrAny", where: str, path: str):
|
||||
def func(s: "DictStrAny") -> None:
|
||||
if is_interpolated_string(s):
|
||||
raise ResolveError(
|
||||
_format_preamble(f"'{where}'", path)
|
||||
+ "interpolating is not allowed"
|
||||
_format_preamble(f"'{where}'", path) + "interpolating is not allowed"
|
||||
)
|
||||
|
||||
return recurse(func)(data)
|
||||
@@ -188,9 +185,7 @@ class DataResolver:
|
||||
def resolve(self):
|
||||
"""Used for testing purposes, otherwise use resolve_one()."""
|
||||
data = join(map(self.resolve_one, self.get_keys()))
|
||||
logger.trace( # type: ignore[attr-defined]
|
||||
"Resolved dvc.yaml:\n%s", data
|
||||
)
|
||||
logger.trace("Resolved dvc.yaml:\n%s", data) # type: ignore[attr-defined]
|
||||
return {STAGES_KWD: data}
|
||||
|
||||
def has_key(self, key: str):
|
||||
@@ -203,9 +198,9 @@ class DataResolver:
|
||||
return False
|
||||
|
||||
if key:
|
||||
return isinstance(
|
||||
definition, ForeachDefinition
|
||||
) and definition.has_member(key)
|
||||
return isinstance(definition, ForeachDefinition) and definition.has_member(
|
||||
key
|
||||
)
|
||||
return not isinstance(definition, ForeachDefinition)
|
||||
|
||||
@collecting
|
||||
@@ -309,9 +304,7 @@ class EntryDefinition:
|
||||
value, skip_interpolation_checks=skip_checks, key=key
|
||||
)
|
||||
except (ParseError, KeyNotInContext) as exc:
|
||||
format_and_raise(
|
||||
exc, f"'{self.where}.{self.name}.{key}'", self.relpath
|
||||
)
|
||||
format_and_raise(exc, f"'{self.where}.{self.name}.{key}'", self.relpath)
|
||||
|
||||
|
||||
class IterationPair(NamedTuple):
|
||||
@@ -354,9 +347,7 @@ class ForeachDefinition:
|
||||
try:
|
||||
iterable = self.context.resolve(self.foreach_data, unwrap=False)
|
||||
except (ContextError, ParseError) as exc:
|
||||
format_and_raise(
|
||||
exc, f"'{self.where}.{self.name}.foreach'", self.relpath
|
||||
)
|
||||
format_and_raise(exc, f"'{self.where}.{self.name}.foreach'", self.relpath)
|
||||
|
||||
# foreach data can be a resolved dictionary/list.
|
||||
self._check_is_map_or_seq(iterable)
|
||||
@@ -381,8 +372,10 @@ class ForeachDefinition:
|
||||
if warn_for:
|
||||
linking_verb = "is" if len(warn_for) == 1 else "are"
|
||||
logger.warning(
|
||||
(
|
||||
"%s %s already specified, "
|
||||
"will be overwritten for stages generated from '%s'",
|
||||
"will be overwritten for stages generated from '%s'"
|
||||
),
|
||||
" and ".join(warn_for),
|
||||
linking_verb,
|
||||
self.name,
|
||||
|
||||
+7
-22
@@ -48,9 +48,7 @@ class MergeError(ContextError):
|
||||
self.key = key
|
||||
to_node = into[key]
|
||||
if not isinstance(to_node, Node) or not isinstance(new, Node):
|
||||
super().__init__(
|
||||
f"cannot merge '{key}' as it already exists in {into}"
|
||||
)
|
||||
super().__init__(f"cannot merge '{key}' as it already exists in {into}")
|
||||
return
|
||||
|
||||
assert isinstance(to_node, Node)
|
||||
@@ -138,9 +136,7 @@ class Node:
|
||||
@dataclass
|
||||
class Value(Node):
|
||||
_value: Any
|
||||
meta: Meta = field(
|
||||
compare=False, default_factory=_default_meta, repr=False
|
||||
)
|
||||
meta: Meta = field(compare=False, default_factory=_default_meta, repr=False)
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self._value)
|
||||
@@ -182,10 +178,7 @@ class Container(Node, ABC): # noqa: B024
|
||||
assert meta
|
||||
container = CtxDict if isinstance(value, dict) else CtxList
|
||||
return container(value, meta=meta)
|
||||
msg = (
|
||||
"Unsupported value of type "
|
||||
f"'{type(value).__name__}' in '{meta}'"
|
||||
)
|
||||
msg = f"Unsupported value of type '{type(value).__name__}' in '{meta}'"
|
||||
raise TypeError(msg)
|
||||
|
||||
def __repr__(self):
|
||||
@@ -219,9 +212,7 @@ class Container(Node, ABC): # noqa: B024
|
||||
try:
|
||||
d = self[index]
|
||||
except LookupError as exc:
|
||||
raise ValueError(
|
||||
f"Could not find '{index}' in {self.data}"
|
||||
) from exc
|
||||
raise ValueError(f"Could not find '{index}' in {self.data}") from exc
|
||||
|
||||
if not rems:
|
||||
return d
|
||||
@@ -380,9 +371,7 @@ class Context(CtxDict):
|
||||
data = {key: data[key] for key in select_keys}
|
||||
except KeyError as exc:
|
||||
key, *_ = exc.args
|
||||
raise ParamsLoadError(
|
||||
f"could not find '{key}' in '{path}'"
|
||||
) from exc
|
||||
raise ParamsLoadError(f"could not find '{key}' in '{path}'") from exc
|
||||
|
||||
meta = Meta(source=path, local=False)
|
||||
ctx = cls(data, meta=meta)
|
||||
@@ -480,9 +469,7 @@ class Context(CtxDict):
|
||||
But for now, just `item` and `key`, this should do.
|
||||
"""
|
||||
# using dict to make the error messages ordered
|
||||
new = dict.fromkeys(
|
||||
[key for key in keys if key not in self._reserved_keys]
|
||||
)
|
||||
new = dict.fromkeys([key for key in keys if key not in self._reserved_keys])
|
||||
self._reserved_keys.update(new)
|
||||
try:
|
||||
yield
|
||||
@@ -542,9 +529,7 @@ class Context(CtxDict):
|
||||
if is_exact_string(src, matches):
|
||||
# replace "${enabled}", if `enabled` is a boolean, with it's actual
|
||||
# value rather than it's string counterparts.
|
||||
expr = get_expression(
|
||||
matches[0], skip_checks=skip_interpolation_checks
|
||||
)
|
||||
expr = get_expression(matches[0], skip_checks=skip_interpolation_checks)
|
||||
value = self.select(expr, unwrap=unwrap)
|
||||
validate_value(value, key)
|
||||
return value
|
||||
|
||||
@@ -99,7 +99,6 @@ def _(obj: dict): # noqa: C901
|
||||
|
||||
result = ""
|
||||
for k, v in flatten(obj).items():
|
||||
|
||||
if isinstance(v, bool):
|
||||
if v:
|
||||
result += f"--{k} "
|
||||
@@ -115,9 +114,7 @@ def _(obj: dict): # noqa: C901
|
||||
if isinstance(i, str):
|
||||
i = escape_str(i)
|
||||
elif isinstance(i, Iterable):
|
||||
raise ParseError(
|
||||
f"Cannot interpolate nested iterable in '{k}'"
|
||||
)
|
||||
raise ParseError(f"Cannot interpolate nested iterable in '{k}'")
|
||||
|
||||
if config.get("list", "nargs") == "append":
|
||||
result += f"--{k} {i} "
|
||||
@@ -206,9 +203,7 @@ def validate_value(value, key):
|
||||
if not_primitive and not_foreach:
|
||||
if isinstance(value, dict) and key == "cmd":
|
||||
return True
|
||||
raise ParseError(
|
||||
f"Cannot interpolate data of type '{type(value).__name__}'"
|
||||
)
|
||||
raise ParseError(f"Cannot interpolate data of type '{type(value).__name__}'")
|
||||
|
||||
|
||||
def str_interpolate(
|
||||
|
||||
@@ -9,9 +9,7 @@ META_KWD = "meta"
|
||||
|
||||
def lockfile_version_schema(value):
|
||||
expected = [LOCKFILE_VERSION.V2.value] # pylint: disable=no-member
|
||||
msg = "invalid schema version {}, expected one of {}".format(
|
||||
value, expected
|
||||
)
|
||||
msg = f"invalid schema version {value}, expected one of {expected}"
|
||||
return validators.Any(*expected, msg=msg)(value)
|
||||
|
||||
|
||||
|
||||
@@ -13,9 +13,7 @@ class ProcessNotFoundError(DvcException):
|
||||
|
||||
class TimeoutExpired(DvcException):
|
||||
def __init__(self, cmd, timeout):
|
||||
super().__init__(
|
||||
f"'{cmd}' did not complete before timeout '{timeout}'"
|
||||
)
|
||||
super().__init__(f"'{cmd}' did not complete before timeout '{timeout}'")
|
||||
self.cmd = cmd
|
||||
self.timeout = timeout
|
||||
|
||||
|
||||
+1
-3
@@ -91,9 +91,7 @@ class ProcessManager:
|
||||
raise UnsupportedSignalError(sig)
|
||||
|
||||
def handle_closed_process():
|
||||
logging.warning(
|
||||
"Process %s had already aborted unexpectedly.", name
|
||||
)
|
||||
logging.warning("Process %s had already aborted unexpectedly.", name)
|
||||
process_info.returncode = -1
|
||||
self[name] = process_info
|
||||
|
||||
|
||||
+3
-11
@@ -28,9 +28,7 @@ class Tqdm(tqdm):
|
||||
"{postfix[info]}{n_fmt}/{total_fmt}"
|
||||
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
|
||||
)
|
||||
BAR_FMT_NOTOTAL = (
|
||||
"{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
|
||||
)
|
||||
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
|
||||
BYTES_DEFAULTS = {
|
||||
"unit": "B",
|
||||
"unit_scale": True,
|
||||
@@ -75,11 +73,7 @@ class Tqdm(tqdm):
|
||||
if not disable:
|
||||
disable = logger.getEffectiveLevel() > level
|
||||
# auto-disable based on TTY
|
||||
if (
|
||||
not disable
|
||||
and not env2bool(DVC_IGNORE_ISATTY)
|
||||
and hasattr(file, "isatty")
|
||||
):
|
||||
if not disable and not env2bool(DVC_IGNORE_ISATTY) and hasattr(file, "isatty"):
|
||||
disable = not file.isatty()
|
||||
super().__init__(
|
||||
iterable=iterable,
|
||||
@@ -95,9 +89,7 @@ class Tqdm(tqdm):
|
||||
if bar_format is None:
|
||||
if self.__len__():
|
||||
self.bar_format = (
|
||||
self.BAR_FMT_DEFAULT_NESTED
|
||||
if self.pos
|
||||
else self.BAR_FMT_DEFAULT
|
||||
self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT
|
||||
)
|
||||
else:
|
||||
self.bar_format = self.BAR_FMT_NOTOTAL
|
||||
|
||||
+1
-3
@@ -24,9 +24,7 @@ def ask(prompt: str, limited_to: Optional[Collection[str]] = None):
|
||||
if answer in limited_to:
|
||||
return answer
|
||||
|
||||
logger.info(
|
||||
"Your response must be one of: %s. Please try again.", limited_to
|
||||
)
|
||||
logger.info("Your response must be one of: %s. Please try again.", limited_to)
|
||||
|
||||
|
||||
def confirm(statement: str) -> bool:
|
||||
|
||||
@@ -2,13 +2,7 @@ import json
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Union
|
||||
|
||||
from dvc.render import (
|
||||
REVISION_FIELD,
|
||||
REVISIONS_KEY,
|
||||
SRC_FIELD,
|
||||
TYPE_KEY,
|
||||
VERSION_FIELD,
|
||||
)
|
||||
from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD
|
||||
from dvc.render.converter.image import ImageConverter
|
||||
from dvc.render.converter.vega import VegaConverter
|
||||
|
||||
|
||||
@@ -81,9 +81,9 @@ def _is_datapoints(lst: List[Dict]):
|
||||
to unexpected behavior
|
||||
"""
|
||||
|
||||
return all(isinstance(item, dict) for item in lst) and set(
|
||||
first(lst).keys()
|
||||
) == {key for keys in lst for key in keys}
|
||||
return all(isinstance(item, dict) for item in lst) and set(first(lst).keys()) == {
|
||||
key for keys in lst for key in keys
|
||||
}
|
||||
|
||||
|
||||
def get_datapoints(file_content: Dict):
|
||||
@@ -195,7 +195,6 @@ class VegaConverter(Converter):
|
||||
return "x"
|
||||
|
||||
def flat_datapoints(self, revision): # noqa: C901
|
||||
|
||||
file2datapoints, properties = self.convert()
|
||||
|
||||
props_update = {}
|
||||
@@ -218,7 +217,7 @@ class VegaConverter(Converter):
|
||||
num_ys = len(ys)
|
||||
if num_xs > 1 and num_xs != num_ys:
|
||||
raise DvcException(
|
||||
f"Cannot have different number of x and y data sources. Found "
|
||||
"Cannot have different number of x and y data sources. Found "
|
||||
f"{num_xs} x and {num_ys} y data sources."
|
||||
)
|
||||
|
||||
@@ -316,9 +315,7 @@ def _update_from_field(
|
||||
source_field = field
|
||||
|
||||
if len(source_datapoints) != len(target_datapoints):
|
||||
raise IndexError(
|
||||
"Source and target datapoints must have the same length"
|
||||
)
|
||||
raise IndexError("Source and target datapoints must have the same length")
|
||||
|
||||
for index, datapoint in enumerate(target_datapoints):
|
||||
source_datapoint = source_datapoints[index]
|
||||
@@ -332,6 +329,5 @@ def _update_from_index(datapoints: List[Dict], new_field: str):
|
||||
|
||||
|
||||
def _update_all(datapoints: List[Dict], update_dict: Dict):
|
||||
|
||||
for datapoint in datapoints:
|
||||
datapoint.update(update_dict)
|
||||
|
||||
+3
-10
@@ -62,7 +62,6 @@ def match_defs_renderers(
|
||||
out=None,
|
||||
templates_dir: Optional["StrPath"] = None,
|
||||
):
|
||||
|
||||
from dvc_render import ImageRenderer, VegaRenderer
|
||||
|
||||
plots_data = PlotsData(data)
|
||||
@@ -80,9 +79,7 @@ def match_defs_renderers(
|
||||
|
||||
for rev, inner_id, plot_definition in group:
|
||||
plot_sources = infer_data_sources(inner_id, plot_definition)
|
||||
definitions_data = plots_data.get_definition_data(
|
||||
plot_sources, rev
|
||||
)
|
||||
definitions_data = plots_data.get_definition_data(plot_sources, rev)
|
||||
|
||||
if ImageRenderer.matches(inner_id, None):
|
||||
renderer_cls = ImageRenderer
|
||||
@@ -91,9 +88,7 @@ def match_defs_renderers(
|
||||
renderer_cls = VegaRenderer
|
||||
renderer_id = plot_id
|
||||
|
||||
converter = _get_converter(
|
||||
renderer_cls, inner_id, props, definitions_data
|
||||
)
|
||||
converter = _get_converter(renderer_cls, inner_id, props, definitions_data)
|
||||
|
||||
dps, rev_props = converter.flat_datapoints(rev)
|
||||
if not final_props and rev_props:
|
||||
@@ -103,7 +98,5 @@ def match_defs_renderers(
|
||||
if "title" not in final_props:
|
||||
final_props["title"] = renderer_id
|
||||
if renderer_cls is not None:
|
||||
renderers.append(
|
||||
renderer_cls(plot_datapoints, renderer_id, **final_props)
|
||||
)
|
||||
renderers.append(renderer_cls(plot_datapoints, renderer_id, **final_props))
|
||||
return renderers
|
||||
|
||||
+7
-27
@@ -3,16 +3,7 @@ import os
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Callable,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple, Union, cast
|
||||
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
@@ -148,9 +139,7 @@ class Repo:
|
||||
|
||||
import hashlib
|
||||
|
||||
root_dir_hash = hashlib.sha224(
|
||||
self.root_dir.encode("utf-8")
|
||||
).hexdigest()
|
||||
root_dir_hash = hashlib.sha224(self.root_dir.encode("utf-8")).hexdigest()
|
||||
|
||||
db_dir = self.fs.path.join(
|
||||
base_db_dir,
|
||||
@@ -230,9 +219,7 @@ class Repo:
|
||||
self.lock = make_lock(
|
||||
self.fs.path.join(self.tmp_dir, "lock"),
|
||||
tmp_dir=self.tmp_dir,
|
||||
hardlink_lock=self.config["core"].get(
|
||||
"hardlink_lock", False
|
||||
),
|
||||
hardlink_lock=self.config["core"].get("hardlink_lock", False),
|
||||
friendly=True,
|
||||
)
|
||||
state_db_dir = self._get_database_dir("state")
|
||||
@@ -338,8 +325,7 @@ class Repo:
|
||||
from dvc.machine import MachineManager
|
||||
|
||||
if self.tmp_dir and (
|
||||
self.config["feature"].get("machine", False)
|
||||
or env2bool("DVC_TEST")
|
||||
self.config["feature"].get("machine", False) or env2bool("DVC_TEST")
|
||||
):
|
||||
return MachineManager(self)
|
||||
return None
|
||||
@@ -393,14 +379,10 @@ class Repo:
|
||||
return os.path.join(root_dir, cls.DVC_DIR)
|
||||
|
||||
@staticmethod
|
||||
def init(
|
||||
root_dir=os.curdir, no_scm=False, force=False, subdir=False
|
||||
) -> "Repo":
|
||||
def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo":
|
||||
from dvc.repo.init import init
|
||||
|
||||
return init(
|
||||
root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir
|
||||
)
|
||||
return init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir)
|
||||
|
||||
def unprotect(self, target):
|
||||
return self.odb.repo.unprotect(target)
|
||||
@@ -530,9 +512,7 @@ class Repo:
|
||||
def dvcfs(self) -> "DVCFileSystem":
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
|
||||
return DVCFileSystem(
|
||||
repo=self, subrepos=self.subrepos, **self._fs_conf
|
||||
)
|
||||
return DVCFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf)
|
||||
|
||||
@cached_property
|
||||
def index_db_dir(self):
|
||||
|
||||
+2
-9
@@ -15,12 +15,7 @@ from dvc.exceptions import (
|
||||
)
|
||||
from dvc.repo.scm_context import scm_context
|
||||
from dvc.ui import ui
|
||||
from dvc.utils import (
|
||||
LARGE_DIR_SIZE,
|
||||
glob_targets,
|
||||
resolve_output,
|
||||
resolve_paths,
|
||||
)
|
||||
from dvc.utils import LARGE_DIR_SIZE, glob_targets, resolve_output, resolve_paths
|
||||
from dvc.utils.collections import ensure_list, validate
|
||||
|
||||
from . import locked
|
||||
@@ -108,9 +103,7 @@ def translate_graph_error(stages: Stages) -> Iterator[None]:
|
||||
def progress_iter(stages: Stages) -> Iterator["Stage"]:
|
||||
total = len(stages)
|
||||
desc = "Adding..."
|
||||
with ui.progress(
|
||||
stages, total=total, desc=desc, unit="file", leave=True
|
||||
) as pbar:
|
||||
with ui.progress(stages, total=total, desc=desc, unit="file", leave=True) as pbar:
|
||||
if total == 1:
|
||||
pbar.bar_format = desc
|
||||
pbar.refresh()
|
||||
|
||||
@@ -49,15 +49,11 @@ def brancher( # noqa: E302
|
||||
|
||||
repo_root_parts = ()
|
||||
if self.fs.path.isin(self.root_dir, self.scm.root_dir):
|
||||
repo_root_parts = self.fs.path.relparts(
|
||||
self.root_dir, self.scm.root_dir
|
||||
)
|
||||
repo_root_parts = self.fs.path.relparts(self.root_dir, self.scm.root_dir)
|
||||
|
||||
cwd_parts = ()
|
||||
if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir):
|
||||
cwd_parts = self.fs.path.relparts(
|
||||
self.fs.path.getcwd(), self.scm.root_dir
|
||||
)
|
||||
cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.scm.root_dir)
|
||||
|
||||
saved_fs = self.fs
|
||||
saved_root = self.root_dir
|
||||
@@ -91,9 +87,7 @@ def brancher( # noqa: E302
|
||||
self.root_dir = self.fs.path.join("/", *repo_root_parts)
|
||||
|
||||
if cwd_parts:
|
||||
cwd = self.fs.path.join( # type: ignore[unreachable]
|
||||
"/", *cwd_parts
|
||||
)
|
||||
cwd = self.fs.path.join("/", *cwd_parts) # type: ignore[unreachable]
|
||||
self.fs.path.chdir(cwd)
|
||||
|
||||
# ignore revs that don't contain repo root
|
||||
|
||||
+3
-12
@@ -2,11 +2,7 @@ import logging
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Dict, List, Set
|
||||
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
CheckoutErrorSuggestGit,
|
||||
NoOutputOrStageError,
|
||||
)
|
||||
from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit, NoOutputOrStageError
|
||||
from dvc.utils import relpath
|
||||
|
||||
from . import locked
|
||||
@@ -35,18 +31,13 @@ def _remove_unused_links(repo):
|
||||
|
||||
|
||||
def get_all_files_numbers(pairs):
|
||||
return sum(
|
||||
stage.get_all_files_number(filter_info) for stage, filter_info in pairs
|
||||
)
|
||||
return sum(stage.get_all_files_number(filter_info) for stage, filter_info in pairs)
|
||||
|
||||
|
||||
def _collect_pairs(
|
||||
self: "Repo", targets, with_deps: bool, recursive: bool
|
||||
) -> Set["StageInfo"]:
|
||||
from dvc.stage.exceptions import (
|
||||
StageFileBadNameError,
|
||||
StageFileDoesNotExistError,
|
||||
)
|
||||
from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
|
||||
|
||||
pairs: Set["StageInfo"] = set()
|
||||
for target in targets:
|
||||
|
||||
+2
-5
@@ -30,8 +30,7 @@ def prompt_to_commit(stage, changes, force=False):
|
||||
|
||||
if not (force or prompt.confirm(_prepare_message(stage, changes))):
|
||||
raise StageCommitError(
|
||||
"unable to commit changed {}. Use `-f|--force` to "
|
||||
"force.".format(stage)
|
||||
f"unable to commit changed {stage}. Use `-f|--force` to force."
|
||||
)
|
||||
|
||||
|
||||
@@ -61,8 +60,6 @@ def commit(
|
||||
if any(changes):
|
||||
prompt_to_commit(stage, changes, force=force)
|
||||
stage.save(allow_missing=allow_missing)
|
||||
stage.commit(
|
||||
filter_info=stage_info.filter_info, allow_missing=allow_missing
|
||||
)
|
||||
stage.commit(filter_info=stage_info.filter_info, allow_missing=allow_missing)
|
||||
stage.dump(update_pipeline=False)
|
||||
return [s.stage for s in stages_info]
|
||||
|
||||
+4
-11
@@ -112,9 +112,7 @@ class GitInfo(TypedDict, total=False):
|
||||
is_dirty: bool
|
||||
|
||||
|
||||
def _git_info(
|
||||
scm: Union["Git", "NoSCM"], untracked_files: str = "all"
|
||||
) -> GitInfo:
|
||||
def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitInfo:
|
||||
from scmrepo.exceptions import SCMError
|
||||
|
||||
from dvc.scm import NoSCM
|
||||
@@ -145,9 +143,7 @@ def _git_info(
|
||||
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
|
||||
from .index import build_data_index
|
||||
|
||||
workspace = build_data_index(
|
||||
repo.index, repo.root_dir, repo.fs, compute_hash=True
|
||||
)
|
||||
workspace = build_data_index(repo.index, repo.root_dir, repo.fs, compute_hash=True)
|
||||
|
||||
return _diff(
|
||||
repo.index.data["repo"],
|
||||
@@ -180,9 +176,7 @@ class Status(TypedDict):
|
||||
git: GitInfo
|
||||
|
||||
|
||||
def _transform_git_paths_to_dvc(
|
||||
repo: "Repo", files: Iterable[str]
|
||||
) -> List[str]:
|
||||
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
|
||||
"""Transform files rel. to Git root to DVC root, and drop outside files."""
|
||||
rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
|
||||
|
||||
@@ -237,8 +231,7 @@ def ls(
|
||||
) -> Iterator[Dict[str, Any]]:
|
||||
targets = targets or [None]
|
||||
pairs = chain.from_iterable(
|
||||
repo.stage.collect_granular(target, recursive=recursive)
|
||||
for target in targets
|
||||
repo.stage.collect_granular(target, recursive=recursive) for target in targets
|
||||
)
|
||||
for stage, filter_info in pairs:
|
||||
for out in stage.filter_outs(filter_info):
|
||||
|
||||
+7
-19
@@ -63,17 +63,13 @@ def diff(
|
||||
targets_paths = None
|
||||
if targets:
|
||||
# convert targets to paths, and capture any missing targets
|
||||
targets_paths, missing_targets[rev] = _targets_to_paths(
|
||||
dvcfs, targets
|
||||
)
|
||||
targets_paths, missing_targets[rev] = _targets_to_paths(dvcfs, targets)
|
||||
|
||||
results[rev] = _paths_checksums(self, targets_paths)
|
||||
|
||||
if targets:
|
||||
# check for overlapping missing targets between a_rev and b_rev
|
||||
for target in set(missing_targets[a_rev]) & set(
|
||||
missing_targets[b_rev]
|
||||
):
|
||||
for target in set(missing_targets[a_rev]) & set(missing_targets[b_rev]):
|
||||
raise PathMissingError(target, self)
|
||||
|
||||
old = results[a_rev]
|
||||
@@ -112,9 +108,7 @@ def diff(
|
||||
if old[path] != new[path]
|
||||
],
|
||||
"renamed": renamed,
|
||||
"not in cache": [
|
||||
{"path": path, "hash": old[path]} for path in missing
|
||||
],
|
||||
"not in cache": [{"path": path, "hash": old[path]} for path in missing],
|
||||
}
|
||||
|
||||
return ret if any(ret.values()) else {}
|
||||
@@ -162,8 +156,7 @@ def _output_paths( # noqa: C901
|
||||
continue
|
||||
|
||||
yield_output = targets is None or any(
|
||||
output.fs.path.isin_or_eq(output.fs_path, target)
|
||||
for target in targets
|
||||
output.fs.path.isin_or_eq(output.fs_path, target) for target in targets
|
||||
)
|
||||
|
||||
obj: Optional["HashFile"]
|
||||
@@ -192,15 +185,12 @@ def _output_paths( # noqa: C901
|
||||
or (
|
||||
isinstance(targets, list)
|
||||
and any(
|
||||
output.fs.path.isin(target, output.fs_path)
|
||||
for target in targets
|
||||
output.fs.path.isin(target, output.fs_path) for target in targets
|
||||
)
|
||||
)
|
||||
):
|
||||
obj = cast("Tree", obj)
|
||||
yield from _dir_output_paths(
|
||||
output.fs, output.fs_path, obj, targets
|
||||
)
|
||||
yield from _dir_output_paths(output.fs, output.fs_path, obj, targets)
|
||||
|
||||
|
||||
def _dir_output_paths(
|
||||
@@ -277,8 +267,6 @@ def _calculate_renamed(
|
||||
continue
|
||||
|
||||
old_path = old_paths.pop(index)
|
||||
renamed.append(
|
||||
{"path": {"old": old_path, "new": path}, "hash": path_hash}
|
||||
)
|
||||
renamed.append({"path": {"old": old_path, "new": path}, "hash": path_hash})
|
||||
|
||||
return renamed
|
||||
|
||||
@@ -279,20 +279,15 @@ class Experiments:
|
||||
branch: Optional[str] = None
|
||||
try:
|
||||
allow_multiple = bool(kwargs.get("params", None))
|
||||
branch = self.get_branch_by_rev(
|
||||
resume_rev, allow_multiple=allow_multiple
|
||||
)
|
||||
branch = self.get_branch_by_rev(resume_rev, allow_multiple=allow_multiple)
|
||||
if not branch:
|
||||
raise DvcException(
|
||||
"Could not find checkpoint experiment "
|
||||
f"'{resume_rev[:7]}'"
|
||||
f"Could not find checkpoint experiment '{resume_rev[:7]}'"
|
||||
)
|
||||
baseline_rev = self._get_baseline(branch)
|
||||
except MultipleBranchError as exc:
|
||||
baselines = {
|
||||
info.baseline_sha
|
||||
for info in exc.ref_infos
|
||||
if info.baseline_sha
|
||||
info.baseline_sha for info in exc.ref_infos if info.baseline_sha
|
||||
}
|
||||
if len(baselines) == 1:
|
||||
baseline_rev = baselines.pop()
|
||||
@@ -341,9 +336,7 @@ class Experiments:
|
||||
self.scm.remove_ref(EXEC_APPLY)
|
||||
|
||||
@unlocked_repo
|
||||
def _reproduce_queue(
|
||||
self, queue: "BaseStashQueue", **kwargs
|
||||
) -> Dict[str, str]:
|
||||
def _reproduce_queue(self, queue: "BaseStashQueue", **kwargs) -> Dict[str, str]:
|
||||
"""Reproduce queued experiments.
|
||||
|
||||
Arguments:
|
||||
@@ -416,9 +409,7 @@ class Experiments:
|
||||
"""
|
||||
result: Dict[str, Optional[str]] = {}
|
||||
exclude = f"{EXEC_NAMESPACE}/*"
|
||||
ref_dict = self.scm.describe(
|
||||
revs, base=EXPS_NAMESPACE, exclude=exclude
|
||||
)
|
||||
ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude)
|
||||
for rev in revs:
|
||||
name: Optional[str] = None
|
||||
ref = ref_dict[rev]
|
||||
|
||||
@@ -8,11 +8,7 @@ from dvc.repo.scm_context import scm_context
|
||||
from dvc.scm import Git
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
from .exceptions import (
|
||||
ApplyConflictError,
|
||||
BaselineMismatchError,
|
||||
InvalidExpRevError,
|
||||
)
|
||||
from .exceptions import ApplyConflictError, BaselineMismatchError, InvalidExpRevError
|
||||
from .executor.base import BaseExecutor
|
||||
from .refs import EXEC_APPLY
|
||||
|
||||
@@ -43,7 +39,9 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901
|
||||
(
|
||||
exp_ref_info,
|
||||
queue_entry,
|
||||
) = exps.celery_queue.get_ref_and_entry_by_names(rev)[rev]
|
||||
) = exps.celery_queue.get_ref_and_entry_by_names(
|
||||
rev
|
||||
)[rev]
|
||||
baseline_sha = repo.scm.get_rev()
|
||||
if exp_ref_info:
|
||||
if baseline_sha != exp_ref_info.baseline_sha:
|
||||
@@ -79,8 +77,7 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901
|
||||
|
||||
repo.scm.set_ref(EXEC_APPLY, exp_rev)
|
||||
logger.info(
|
||||
"Changes for experiment '%s' have been applied to your current "
|
||||
"workspace.",
|
||||
"Changes for experiment '%s' have been applied to your current workspace.",
|
||||
rev,
|
||||
)
|
||||
|
||||
|
||||
@@ -33,9 +33,11 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs):
|
||||
break
|
||||
if not ref_info:
|
||||
msg = [
|
||||
(
|
||||
f"Ambiguous experiment name '{exp_rev}' can refer to "
|
||||
"multiple experiments. To create a branch use a full "
|
||||
"experiment ref:",
|
||||
"experiment ref:"
|
||||
),
|
||||
"",
|
||||
]
|
||||
msg.extend([str(info) for info in ref_infos])
|
||||
@@ -46,9 +48,7 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs):
|
||||
|
||||
branch_ref = f"refs/heads/{branch_name}"
|
||||
if repo.scm.get_ref(branch_ref):
|
||||
raise InvalidArgumentError(
|
||||
f"Git branch '{branch_name}' already exists."
|
||||
)
|
||||
raise InvalidArgumentError(f"Git branch '{branch_name}' already exists.")
|
||||
|
||||
target = repo.scm.get_ref(str(ref_info))
|
||||
repo.scm.set_ref(
|
||||
|
||||
@@ -23,9 +23,7 @@ def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs):
|
||||
rev = resolve_rev(repo.scm, b_rev)
|
||||
new = collect_experiment_commit(repo, rev, param_deps=param_deps)
|
||||
else:
|
||||
new = collect_experiment_commit(
|
||||
repo, "workspace", param_deps=param_deps
|
||||
)
|
||||
new = collect_experiment_commit(repo, "workspace", param_deps=param_deps)
|
||||
|
||||
with_unchanged = kwargs.pop("all", False)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user