nit: fit statement/expr in the same line instead of hanging (#10267)

The code are wrapped due to us using line-length=79 before, but now some
of the code can easily fit in 88 chars.

The formatter did not change this automatically when we changed line-length,
because of magic trailing commas that gets added when they turn into multiple lines.

Black/ruff wrapping them in parens and splitting them into multiple lines
was making it difficult to read.
This commit is contained in:
skshetry
2024-01-30 18:23:04 +05:45
committed by GitHub
parent 422caa13c4
commit 9a1bcd7f1d
144 changed files with 342 additions and 1707 deletions
+1 -4
View File
@@ -5,10 +5,7 @@ from PyInstaller.utils.hooks import collect_submodules, is_module_or_submodule
# Celery dynamically imports most celery internals at runtime
# pyinstaller hook must expose all modules loaded by
# kombu.utils.imports:symbol_by_name()
_EXCLUDES = (
"celery.bin",
"celery.contrib",
)
_EXCLUDES = ("celery.bin", "celery.contrib")
hiddenimports = collect_submodules(
"celery",
filter=lambda name: not any(
+2 -10
View File
@@ -57,11 +57,7 @@ def get_url(path, repo=None, rev=None, remote=None):
class _OpenContextManager(GCM):
def __init__(self, func, args, kwds):
self.gen = func(*args, **kwds)
self.func, self.args, self.kwds = ( # type: ignore[assignment]
func,
args,
kwds,
)
self.func, self.args, self.kwds = (func, args, kwds) # type: ignore[assignment]
def __getattr__(self, name):
raise AttributeError("dvc.api.open() should be used in a with statement.")
@@ -262,11 +258,7 @@ def _open(
fs_path = fs.from_os_path(path)
try:
with fs.open(
fs_path,
mode=mode,
encoding=encoding,
) as fobj:
with fs.open(fs_path, mode=mode, encoding=encoding) as fobj:
yield fobj
except FileNotFoundError as exc:
raise FileMissingError(path) from exc
+2 -8
View File
@@ -144,15 +144,9 @@ def migrate_2_to_3(repo: "Repo", dry: bool = False):
)
return
with TqdmCallback(
desc="Computing DVC 3.0 hashes",
unit="files",
) as cb:
with TqdmCallback(desc="Computing DVC 3.0 hashes", unit="files") as cb:
migration = prepare(src, dest, callback=cb)
with TqdmCallback(
desc="Migrating to DVC 3.0 cache",
unit="files",
) as cb:
with TqdmCallback(desc="Migrating to DVC 3.0 cache", unit="files") as cb:
count = migrate(migration, callback=cb)
ui.write(f"Migrated {count} files to DVC 3.0 cache location.")
+1 -5
View File
@@ -107,8 +107,4 @@ def _add_run_common(parser):
default=None,
help="Custom commit message to use when committing the experiment.",
)
parser.add_argument(
"-M", # obsolete
dest="message",
help=argparse.SUPPRESS,
)
parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
+1 -5
View File
@@ -93,9 +93,5 @@ def add_parser(experiments_subparsers, parent_parser):
default=None,
help="Custom commit message to use when committing the experiment.",
)
save_parser.add_argument(
"-M", # obsolete
dest="message",
help=argparse.SUPPRESS,
)
save_parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
save_parser.set_defaults(func=CmdExperimentsSave)
+1 -5
View File
@@ -119,11 +119,7 @@ def add_parser(subparsers, parent_parser):
"specified by '--remote') in the target repository."
),
)
list_parser.add_argument(
"--size",
action="store_true",
help="Show sizes.",
)
list_parser.add_argument("--size", action="store_true", help="Show sizes.")
list_parser.add_argument(
"path",
nargs="?",
+2 -9
View File
@@ -38,16 +38,9 @@ def add_parser(subparsers, parent_parser):
"url", help="See `dvc import-url -h` for full list of supported URLs."
)
lsurl_parser.add_argument(
"-R",
"--recursive",
action="store_true",
help="Recursively list files.",
)
lsurl_parser.add_argument(
"--size",
action="store_true",
help="Show sizes.",
"-R", "--recursive", action="store_true", help="Recursively list files."
)
lsurl_parser.add_argument("--size", action="store_true", help="Show sizes.")
lsurl_parser.add_argument(
"--fs-config",
type=str,
+3 -15
View File
@@ -34,21 +34,12 @@ def _show_json(
name = renderer.name
data[name] = to_json(renderer, split)
all_errors.extend(
{
"name": name,
"rev": rev,
"source": source,
**encode_exception(e),
}
{"name": name, "rev": rev, "source": source, **encode_exception(e)}
for rev, per_rev_src_errors in src_errors.items()
for source, e in per_rev_src_errors.items()
)
all_errors.extend(
{
"name": name,
"rev": rev,
**encode_exception(e),
}
{"name": name, "rev": rev, **encode_exception(e)}
for rev, e in def_errors.items()
)
@@ -101,10 +92,7 @@ class CmdPlots(CmdBase):
return 1
try:
plots_data = self._func(
targets=self.args.targets,
props=self._props(),
)
plots_data = self._func(targets=self.args.targets, props=self._props())
if not plots_data and not self.args.json:
ui.error_write(
+1 -8
View File
@@ -2,14 +2,7 @@ from dvc.cli import formatter
from dvc.cli.utils import append_doc_link
from dvc.commands.queue import kill, logs, remove, start, status, stop
SUB_COMMANDS = [
start,
stop,
status,
logs,
remove,
kill,
]
SUB_COMMANDS = [start, stop, status, logs, remove, kill]
def add_parser(subparsers, parent_parser):
+1 -5
View File
@@ -45,9 +45,5 @@ def add_parser(queue_subparsers, parent_parser):
),
action="store_true",
)
queue_logs_parser.add_argument(
"task",
help="Task to show.",
metavar="<task>",
)
queue_logs_parser.add_argument("task", help="Task to show.", metavar="<task>")
queue_logs_parser.set_defaults(func=CmdQueueLogs)
+1 -7
View File
@@ -9,13 +9,7 @@ from collections.abc import (
)
from itertools import chain, repeat, zip_longest
from operator import itemgetter
from typing import (
TYPE_CHECKING,
Any,
Optional,
Union,
overload,
)
from typing import TYPE_CHECKING, Any, Optional, Union, overload
from funcy import reraise
+1 -6
View File
@@ -107,12 +107,7 @@ def _posix_detached_subprocess(args: Sequence[str], **kwargs) -> int:
os.close(read_end)
return int(pid_str)
proc = subprocess.Popen(
args,
shell=False, # noqa: S603
close_fds=True,
**kwargs,
)
proc = subprocess.Popen(args, shell=False, close_fds=True, **kwargs) # noqa: S603
os.close(read_end)
os.write(write_end, str(proc.pid).encode("utf8"))
os.close(write_end)
+1 -9
View File
@@ -59,15 +59,7 @@ def loads_from(stage, s_list, erepo=None, fs_config=None, db=None):
info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
if db:
info.update({"db": db})
return [
_get(
stage,
s,
info.copy(),
fs_config=fs_config,
)
for s in s_list
]
return [_get(stage, s, info.copy(), fs_config=fs_config) for s in s_list]
def _merge_params(s_list) -> dict[str, list[str]]:
+2 -8
View File
@@ -76,10 +76,7 @@ class ParamsDependency(Dependency):
self.params = list(params) if params else []
hash_info = HashInfo()
if isinstance(params, dict):
hash_info = HashInfo(
self.PARAM_PARAMS,
params, # type: ignore[arg-type]
)
hash_info = HashInfo(self.PARAM_PARAMS, params) # type: ignore[arg-type]
repo = repo or stage.repo
path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE)
super().__init__(stage, path, repo=repo)
@@ -103,10 +100,7 @@ class ParamsDependency(Dependency):
for param in self.params:
if param in values:
info[param] = values[param]
self.hash_info = HashInfo(
self.PARAM_PARAMS,
info, # type: ignore[arg-type]
)
self.hash_info = HashInfo(self.PARAM_PARAMS, info) # type: ignore[arg-type]
def read_params(
self, flatten: bool = True, **kwargs: typing.Any
+1 -9
View File
@@ -1,14 +1,6 @@
import contextlib
import os
from typing import (
TYPE_CHECKING,
Any,
Callable,
ClassVar,
Optional,
TypeVar,
Union,
)
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, TypeVar, Union
from dvc.exceptions import DvcException
from dvc.log import logger
+2 -12
View File
@@ -52,10 +52,7 @@ def download(
from .callbacks import TqdmCallback
with TqdmCallback(
desc=f"Downloading {fs.name(fs_path)}",
unit="files",
) as cb:
with TqdmCallback(desc=f"Downloading {fs.name(fs_path)}", unit="files") as cb:
# NOTE: We use dvc-objects generic.copy over fs.get since it makes file
# download atomic and avoids fsspec glob/regex path expansion.
if fs.isdir(fs_path):
@@ -76,14 +73,7 @@ def download(
lfs_prefetch(fs, from_infos)
cb.set_size(len(from_infos))
jobs = jobs or fs.jobs
generic.copy(
fs,
from_infos,
localfs,
to_infos,
callback=cb,
batch_size=jobs,
)
generic.copy(fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs)
return len(to_infos)
+1 -3
View File
@@ -22,9 +22,7 @@ class DataFileSystem(FileSystem):
return config
@functools.cached_property
def fs(
self,
) -> "_DataFileSystem":
def fs(self) -> "_DataFileSystem":
from dvc_data.fs import DataFileSystem as _DataFileSystem
return _DataFileSystem(**self.fs_args)
+1 -3
View File
@@ -39,9 +39,7 @@ class GitFileSystem(FileSystem):
)
@functools.cached_property
def fs(
self,
) -> "FsspecGitFileSystem":
def fs(self) -> "FsspecGitFileSystem":
from scmrepo.fs import GitFileSystem as FsspecGitFileSystem
return FsspecGitFileSystem(**self.fs_args)
+1 -3
View File
@@ -185,9 +185,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
if self._tmp_dir is not None:
# Under Windows file path length is limited so we hash it
hasher = hashlib.md5( # noqa: S324
self._claimfile.encode()
)
hasher = hashlib.md5(self._claimfile.encode()) # noqa: S324
filename = hasher.hexdigest()
self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
+9 -43
View File
@@ -167,11 +167,7 @@ def _merge_data(s_list):
@collecting
def load_from_pipeline(stage, data, typ="outs"):
if typ not in (
stage.PARAM_OUTS,
stage.PARAM_METRICS,
stage.PARAM_PLOTS,
):
if typ not in (stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS):
raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
metric = typ == stage.PARAM_METRICS
@@ -197,24 +193,13 @@ def load_from_pipeline(stage, data, typ="outs"):
],
)
yield _get(
stage,
path,
info={},
plot=plt_d or plot,
metric=metric,
**extra,
)
yield _get(stage, path, info={}, plot=plt_d or plot, metric=metric, **extra)
def split_file_meta_from_cloud(entry: dict) -> dict:
if remote_name := entry.pop(Meta.PARAM_REMOTE, None):
remote_meta = {}
for key in (
S3_PARAM_CHECKSUM,
HDFS_PARAM_CHECKSUM,
Meta.PARAM_VERSION_ID,
):
for key in (S3_PARAM_CHECKSUM, HDFS_PARAM_CHECKSUM, Meta.PARAM_VERSION_ID):
if value := entry.pop(key, None):
remote_meta[key] = value
@@ -433,10 +418,7 @@ class Output:
hash_name = meta_name = self.fs.PARAM_CHECKSUM
assert hash_name
hash_info = HashInfo(
name=hash_name,
value=getattr(self.meta, meta_name, None),
)
hash_info = HashInfo(name=hash_name, value=getattr(self.meta, meta_name, None))
return hash_name, hash_info
def _compute_meta_hash_info_from_files(self) -> None:
@@ -515,10 +497,7 @@ class Output:
if self.fs.isabs(self.def_path):
return False
return self.repo and self.fs.isin(
self.fs_path,
self.repo.root_dir,
)
return self.repo and self.fs.isin(self.fs_path, self.repo.root_dir)
@property
def use_scm_ignore(self):
@@ -813,20 +792,13 @@ class Output:
def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
staging, _, obj = self._build(
self.cache,
self.fs_path,
self.fs,
self.hash_name,
ignore=self.dvcignore,
self.cache, self.fs_path, self.fs, self.hash_name, ignore=self.dvcignore
)
assert isinstance(obj, Tree)
save_obj = obj.filter(prefix)
assert isinstance(save_obj, Tree)
checkout_obj = save_obj.get_obj(self.cache, prefix)
with TqdmCallback(
desc=f"Committing {self} to cache",
unit="file",
) as cb:
with TqdmCallback(desc=f"Committing {self} to cache", unit="file") as cb:
otransfer(
staging,
self.cache,
@@ -1286,10 +1258,7 @@ class Output:
self.hash_info = merged.hash_info
self.files = None
self.meta = Meta(
size=du(self.cache, merged),
nfiles=len(merged),
)
self.meta = Meta(size=du(self.cache, merged), nfiles=len(merged))
def unstage(self, path: str) -> tuple["Meta", "Tree"]:
from pygtrie import Trie
@@ -1429,10 +1398,7 @@ class Output:
assert staging
assert obj.hash_info
with TqdmCallback(
desc=f"Adding {self} to cache",
unit="file",
) as cb:
with TqdmCallback(desc=f"Adding {self} to cache", unit="file") as cb:
otransfer(
staging,
self.cache,
+1 -7
View File
@@ -3,13 +3,7 @@ import os
from collections.abc import Mapping, Sequence
from copy import deepcopy
from itertools import product
from typing import (
TYPE_CHECKING,
Any,
NamedTuple,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
from funcy import collecting, first, isa, join, reraise
+2 -8
View File
@@ -28,9 +28,7 @@ class ImageConverter(Converter):
return img_path
@staticmethod
def _encode_image(
image_data: bytes,
) -> str:
def _encode_image(image_data: bytes) -> str:
base64_str = base64.b64encode(image_data).decode()
return f"data:image;base64,{base64_str}"
@@ -57,10 +55,6 @@ class ImageConverter(Converter):
)
else:
src = self._encode_image(image_data)
datapoint = {
REVISION: revision,
FILENAME: filename,
SRC: src,
}
datapoint = {REVISION: revision, FILENAME: filename, SRC: src}
datapoints.append(datapoint)
return datapoints, properties
+4 -16
View File
@@ -41,11 +41,7 @@ def _file_field(*args):
yield file, field
def _find(
filename: str,
field: str,
data_series: list[tuple[str, str, Any]],
):
def _find(filename: str, field: str, data_series: list[tuple[str, str, Any]]):
for data_file, data_field, data in data_series:
if data_file == filename and data_field == field:
return data_file, data_field, data
@@ -202,10 +198,7 @@ class VegaConverter(Converter):
# assign "step" if no x provided
if not xs:
x_file, x_field = (
None,
INDEX,
)
x_file, x_field = None, INDEX
else:
x_file, x_field = xs[0]
@@ -244,10 +237,7 @@ class VegaConverter(Converter):
common_prefix_len = 0
props_update["anchors_y_definitions"] = [
{
FILENAME: _get_short_y_file(y_file, common_prefix_len),
FIELD: y_field,
}
{FILENAME: _get_short_y_file(y_file, common_prefix_len), FIELD: y_field}
for y_file, y_field in ys
]
@@ -299,9 +289,7 @@ class VegaConverter(Converter):
return all_datapoints, properties
def convert(
self,
):
def convert(self):
"""
Convert the data. Fill necessary fields ('x', 'y') and return both
generated datapoints and updated properties. `x`, `y` values and labels
+5 -28
View File
@@ -3,12 +3,7 @@ from collections import defaultdict
from collections.abc import Iterable
from contextlib import AbstractContextManager, contextmanager
from functools import wraps
from typing import (
TYPE_CHECKING,
Callable,
Optional,
Union,
)
from typing import TYPE_CHECKING, Callable, Optional, Union
from dvc.exceptions import (
DvcException,
@@ -179,14 +174,8 @@ class Repo:
self.root_dir: str
self.dvc_dir: Optional[str]
(
self.root_dir,
self.dvc_dir,
) = self._get_repo_dirs(
root_dir=root_dir,
fs=self.fs,
uninitialized=uninitialized,
scm=scm,
(self.root_dir, self.dvc_dir) = self._get_repo_dirs(
root_dir=root_dir, fs=self.fs, uninitialized=uninitialized, scm=scm
)
self._uninitialized = uninitialized
@@ -271,11 +260,7 @@ class Repo:
# subrepo
relparts = self.fs.relparts(self.root_dir, "/")
dvc_dir = os.path.join(
self.scm.root_dir,
*relparts,
self.DVC_DIR,
)
dvc_dir = os.path.join(self.scm.root_dir, *relparts, self.DVC_DIR)
if os.path.exists(dvc_dir):
return dvc_dir
@@ -651,15 +636,7 @@ class Repo:
btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None)
md5 = hashlib.md5( # noqa: S324
str(
(
root_dir,
btime,
getpass.getuser(),
version_tuple[0],
salt,
)
).encode()
str((root_dir, btime, getpass.getuser(), version_tuple[0], salt)).encode()
)
repo_token = md5.hexdigest()
return os.path.join(repos_dir, repo_token)
+1 -6
View File
@@ -1,12 +1,7 @@
import os
from collections.abc import Iterator
from contextlib import contextmanager
from typing import (
TYPE_CHECKING,
NamedTuple,
Optional,
Union,
)
from typing import TYPE_CHECKING, NamedTuple, Optional, Union
from dvc.exceptions import (
CacheLinkError,
+2 -10
View File
@@ -98,10 +98,7 @@ class Artifacts:
def read(self) -> dict[str, dict[str, Artifact]]:
"""Read artifacts from dvc.yaml."""
artifacts: dict[str, dict[str, Artifact]] = {}
for (
dvcfile,
dvcfile_artifacts,
) in self.repo.index._artifacts.items():
for dvcfile, dvcfile_artifacts in self.repo.index._artifacts.items():
dvcyaml = relpath(dvcfile, self.repo.root_dir)
artifacts[dvcyaml] = {}
for name, value in dvcfile_artifacts.items():
@@ -188,12 +185,7 @@ class Artifacts:
out = resolve_output(path, out, force=force)
fs = self.repo.dvcfs
fs_path = fs.from_os_path(path)
count = fs_download(
fs,
fs_path,
os.path.abspath(out),
jobs=jobs,
)
count = fs_download(fs, fs_path, os.path.abspath(out), jobs=jobs)
return count, out
@staticmethod
+1 -5
View File
@@ -5,11 +5,7 @@ def check_missing(repo, rev=None, max_size=None, types=None):
from dvc_data.index import StorageKeyError
with repo.switch(rev or "workspace"):
idx = repo.index.targets_view(
None,
max_size=max_size,
types=types,
)
idx = repo.index.targets_view(None, max_size=max_size, types=types)
index = idx.data["repo"]
+6 -29
View File
@@ -122,42 +122,23 @@ def checkout( # noqa: C901
def onerror(target, exc):
if target and isinstance(
exc,
(
StageFileDoesNotExistError,
StageFileBadNameError,
NoOutputOrStageError,
),
(StageFileDoesNotExistError, StageFileBadNameError, NoOutputOrStageError),
):
raise CheckoutErrorSuggestGit(target) from exc
raise
view = self.index.targets_view(
targets,
recursive=recursive,
with_deps=with_deps,
onerror=onerror,
targets, recursive=recursive, with_deps=with_deps, onerror=onerror
)
with ui.progress(
unit="entry",
desc="Building workspace index",
leave=True,
) as pb:
with ui.progress(unit="entry", desc="Building workspace index", leave=True) as pb:
old = build_data_index(
view,
self.root_dir,
self.fs,
compute_hash=True,
callback=pb.as_callback(),
view, self.root_dir, self.fs, compute_hash=True, callback=pb.as_callback()
)
new = view.data["repo"]
with ui.progress(
desc="Comparing indexes",
unit="entry",
leave=True,
) as pb:
with ui.progress(desc="Comparing indexes", unit="entry", leave=True) as pb:
diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback())
if not force:
@@ -175,11 +156,7 @@ def checkout( # noqa: C901
if self.fs.isin_or_eq(dest_path, out_path):
failed.add(out_path)
with ui.progress(
unit="file",
desc="Applying changes",
leave=True,
) as pb:
with ui.progress(unit="file", desc="Applying changes", leave=True) as pb:
apply(
diff,
self.root_dir,
+4 -17
View File
@@ -154,10 +154,7 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
from .index import build_data_index
with ui.progress(
desc="Building workspace index",
unit="entry",
) as pb:
with ui.progress(desc="Building workspace index", unit="entry") as pb:
workspace = build_data_index(
repo.index,
repo.root_dir,
@@ -187,10 +184,7 @@ def _diff_head_to_index(
with repo.switch(head):
head_index = repo.index.data["repo"]
with ui.progress(
desc="Calculating diff between head/index",
unit="entry",
) as pb:
with ui.progress(desc="Calculating diff between head/index", unit="entry") as pb:
return _diff(head_index, index, callback=pb.as_callback(), **kwargs)
@@ -223,18 +217,11 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
return [repo.fs.relpath(file, start) for file in files]
def status(
repo: "Repo",
untracked_files: str = "no",
**kwargs: Any,
) -> Status:
def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
from dvc.scm import NoSCMError, SCMError
head = kwargs.pop("head", "HEAD")
uncommitted_diff = _diff_index_to_wtree(
repo,
**kwargs,
)
uncommitted_diff = _diff_index_to_wtree(repo, **kwargs)
unchanged = set(uncommitted_diff.pop("unchanged", []))
try:
+7 -33
View File
@@ -52,36 +52,22 @@ def _diff(old, new, data_keys, with_missing=False):
continue
if change.typ == ADD:
ret["added"].append(
{
"path": _path(change.new),
"hash": _hash(change.new),
}
)
ret["added"].append({"path": _path(change.new), "hash": _hash(change.new)})
elif change.typ == DELETE:
ret["deleted"].append(
{
"path": _path(change.old),
"hash": _hash(change.old),
}
{"path": _path(change.old), "hash": _hash(change.old)}
)
elif change.typ == MODIFY:
ret["modified"].append(
{
"path": _path(change.old),
"hash": {
"old": _hash(change.old),
"new": _hash(change.new),
},
"hash": {"old": _hash(change.old), "new": _hash(change.new)},
}
)
elif change.typ == RENAME:
ret["renamed"].append(
{
"path": {
"old": _path(change.old),
"new": _path(change.new),
},
"path": {"old": _path(change.old), "new": _path(change.new)},
"hash": _hash(change.old),
}
)
@@ -93,10 +79,7 @@ def _diff(old, new, data_keys, with_missing=False):
and not old.storage_map.cache_exists(change.old)
):
ret["not in cache"].append(
{
"path": _path(change.old),
"hash": _hash(change.old),
}
{"path": _path(change.old), "hash": _hash(change.old)}
)
return ret if any(ret.values()) else {}
@@ -137,11 +120,7 @@ def diff(
def onerror(target, _exc):
missing_targets[rev].add(target) # noqa: B023
view = self.index.targets_view(
targets,
onerror=onerror,
recursive=recursive,
)
view = self.index.targets_view(targets, onerror=onerror, recursive=recursive)
data_keys.update(view.data_keys.get("repo", set()))
@@ -149,12 +128,7 @@ def diff(
from .index import build_data_index
with ui.status("Building workspace index"):
data = build_data_index(
view,
self.root_dir,
self.fs,
compute_hash=True,
)
data = build_data_index(view, self.root_dir, self.fs, compute_hash=True)
else:
data = view.data["repo"]
+5 -23
View File
@@ -131,16 +131,9 @@ class Experiments:
self._log_reproduced(results, tmp_dir=tmp_dir)
return results
def queue_one(
self,
queue: "BaseStashQueue",
**kwargs,
) -> "QueueEntry":
def queue_one(self, queue: "BaseStashQueue", **kwargs) -> "QueueEntry":
"""Queue a single experiment."""
return self.new(
queue,
**kwargs,
)
return self.new(queue, **kwargs)
def reproduce_celery(
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
@@ -148,16 +141,10 @@ class Experiments:
results: dict[str, str] = {}
if entries is None:
entries = list(
chain(
self.celery_queue.iter_active(),
self.celery_queue.iter_queued(),
)
chain(self.celery_queue.iter_active(), self.celery_queue.iter_queued())
)
logger.debug(
"reproduce all these entries '%s'",
entries,
)
logger.debug("reproduce all these entries '%s'", entries)
if not entries:
return results
@@ -212,12 +199,7 @@ class Experiments:
else:
ui.write("Experiment results have been applied to your workspace.")
def new(
self,
queue: "BaseStashQueue",
*args,
**kwargs,
) -> "QueueEntry":
def new(self, queue: "BaseStashQueue", *args, **kwargs) -> "QueueEntry":
"""Create and enqueue a new experiment.
Experiment will be derived from the current workspace.
+3 -4
View File
@@ -33,10 +33,9 @@ def apply(repo: "Repo", rev: str, **kwargs):
try:
exp_rev = resolve_rev(repo.scm, rev)
except RevError as exc:
(
exp_ref_info,
queue_entry,
) = exps.celery_queue.get_ref_and_entry_by_names(rev)[rev]
(exp_ref_info, queue_entry) = exps.celery_queue.get_ref_and_entry_by_names(rev)[
rev
]
if exp_ref_info:
exp_rev = repo.scm.get_ref(str(exp_ref_info))
elif queue_entry:
+1 -4
View File
@@ -10,10 +10,7 @@ if TYPE_CHECKING:
@contextmanager
def switch_repo(
repo: "Repo",
rev: str,
) -> Iterator[tuple["Repo", str]]:
def switch_repo(repo: "Repo", rev: str) -> Iterator[tuple["Repo", str]]:
"""Return a repo instance (brancher) switched to rev.
If rev is the name of a running experiment, the returned instance will be
+2 -12
View File
@@ -3,11 +3,7 @@ import os
from collections.abc import Collection, Iterable, Iterator
from dataclasses import fields
from datetime import datetime
from typing import (
TYPE_CHECKING,
Optional,
Union,
)
from typing import TYPE_CHECKING, Optional, Union
from funcy import first
from scmrepo.exceptions import SCMError as InnerSCMError
@@ -58,13 +54,7 @@ def collect_rev(
else:
orig_cwd = None
try:
data = _collect_rev(
repo,
rev,
param_deps=param_deps,
force=force,
**kwargs,
)
data = _collect_rev(repo, rev, param_deps=param_deps, force=force, **kwargs)
if not (rev == "workspace" or param_deps or data.contains_error):
cache.put(data, force=True)
return ExpState(rev=rev, data=data)
+1 -5
View File
@@ -51,11 +51,7 @@ class MultipleBranchError(DvcException):
class AmbiguousExpRefInfo(InvalidArgumentError):
def __init__(
self,
exp_name: str,
exp_ref_list: Iterable["ExpRefInfo"],
):
def __init__(self, exp_name: str, exp_ref_list: Iterable["ExpRefInfo"]):
msg = [
(
f"Ambiguous name '{exp_name}' refers to multiple experiments."
+1 -8
View File
@@ -8,14 +8,7 @@ from contextlib import contextmanager
from dataclasses import asdict, dataclass
from enum import IntEnum
from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
Callable,
NamedTuple,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
from scmrepo.exceptions import SCMError
+2 -11
View File
@@ -132,11 +132,7 @@ class TempDirExecutor(BaseLocalExecutor):
self._update_config(repo.config.read("local"))
def _update_config(self, update):
local_config = os.path.join(
self.root_dir,
self.dvc_dir,
"config.local",
)
local_config = os.path.join(self.root_dir, self.dvc_dir, "config.local")
logger.debug("Writing experiments local config '%s'", local_config)
if os.path.exists(local_config):
conf_obj = ConfigObj(local_config)
@@ -188,12 +184,7 @@ class WorkspaceExecutor(BaseLocalExecutor):
self._detach_stack = ExitStack()
@classmethod
def from_stash_entry(
cls,
repo: "Repo",
entry: "ExpStashEntry",
**kwargs,
):
def from_stash_entry(cls, repo: "Repo", entry: "ExpStashEntry", **kwargs):
root_dir = repo.scm.root_dir
executor: "WorkspaceExecutor" = cls._from_stash_entry(
repo, entry, root_dir, **kwargs
+1 -6
View File
@@ -1,10 +1,5 @@
from collections.abc import Iterable, Mapping
from typing import (
TYPE_CHECKING,
Any,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, Optional, Union
from funcy import compact, group_by
from scmrepo.git.backend.base import SyncStatus
+4 -20
View File
@@ -2,13 +2,7 @@ import os
from abc import ABC, abstractmethod
from collections.abc import Collection, Generator, Iterable, Mapping
from dataclasses import asdict, dataclass
from typing import (
TYPE_CHECKING,
Any,
NamedTuple,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
from dvc_studio_client.post_live_metrics import get_studio_config
from funcy import retry
@@ -275,12 +269,7 @@ class BaseStashQueue(ABC):
"""
@abstractmethod
def logs(
self,
rev: str,
encoding: Optional[str] = None,
follow: bool = False,
):
def logs(self, rev: str, encoding: Optional[str] = None, follow: bool = False):
"""Print redirected output logs for an exp process.
Args:
@@ -336,9 +325,7 @@ class BaseStashQueue(ABC):
self._stash_commit_deps(*args, **kwargs)
# save additional repro command line arguments
run_env = {
DVC_EXP_BASELINE_REV: baseline_rev,
}
run_env = {DVC_EXP_BASELINE_REV: baseline_rev}
if not name:
name = get_random_exp_name(self.scm, baseline_rev)
run_env[DVC_EXP_NAME] = name
@@ -502,10 +489,7 @@ class BaseStashQueue(ABC):
@staticmethod
@retry(180, errors=LockError, timeout=1)
def get_stash_entry(
exp: "Experiments",
queue_entry: QueueEntry,
) -> "ExpStashEntry":
def get_stash_entry(exp: "Experiments", queue_entry: QueueEntry) -> "ExpStashEntry":
stash = ExpStash(exp.scm, queue_entry.stash_ref)
stash_rev = queue_entry.stash_rev
with get_exp_rwlock(exp.repo, writes=[queue_entry.stash_ref]):
+7 -30
View File
@@ -4,12 +4,7 @@ import logging
import os
from collections import defaultdict
from collections.abc import Collection, Generator, Mapping
from typing import (
TYPE_CHECKING,
NamedTuple,
Optional,
Union,
)
from typing import TYPE_CHECKING, NamedTuple, Optional, Union
from celery.result import AsyncResult
from funcy import first
@@ -83,10 +78,7 @@ class LocalCeleryQueue(BaseStashQueue):
"dvc-exp-local",
wdir=self.wdir,
mkdir=True,
include=[
"dvc.repo.experiments.queue.tasks",
"dvc_task.proc.tasks",
],
include=["dvc.repo.experiments.queue.tasks", "dvc_task.proc.tasks"],
)
app.conf.update({"task_acks_late": True, "result_expires": None})
return app
@@ -379,10 +371,7 @@ class LocalCeleryQueue(BaseStashQueue):
raise CannotKillTasksError(remained_revs)
def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None:
logger.debug(
"Found active tasks: '%s' to kill",
list(entries.values()),
)
logger.debug("Found active tasks: '%s' to kill", list(entries.values()))
inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks(
entries, force
)
@@ -418,20 +407,11 @@ class LocalCeleryQueue(BaseStashQueue):
if to_kill:
self._kill_entries(to_kill, True)
def follow(
self,
entry: QueueEntry,
encoding: Optional[str] = None,
):
def follow(self, entry: QueueEntry, encoding: Optional[str] = None):
for line in self.proc.follow(entry.stash_rev, encoding):
ui.write(line, end="")
def logs(
self,
rev: str,
encoding: Optional[str] = None,
follow: bool = False,
):
def logs(self, rev: str, encoding: Optional[str] = None, follow: bool = False):
queue_entry: Optional[QueueEntry] = self.match_queue_entry_by_name(
{rev}, self.iter_active(), self.iter_done()
).get(rev)
@@ -458,8 +438,7 @@ class LocalCeleryQueue(BaseStashQueue):
f"No output logs found for experiment '{rev}'"
)
with open(
proc_info.stdout,
encoding=encoding or locale.getpreferredencoding(),
proc_info.stdout, encoding=encoding or locale.getpreferredencoding()
) as fobj:
ui.write(fobj.read())
@@ -560,9 +539,7 @@ class LocalCeleryQueue(BaseStashQueue):
return result
def collect_queued_data(
self,
baseline_revs: Optional[Collection[str]],
**kwargs,
self, baseline_revs: Optional[Collection[str]], **kwargs
) -> dict[str, list["ExpRange"]]:
from dvc.repo.experiments.collect import collect_rev
from dvc.repo.experiments.serialize import (
+3 -12
View File
@@ -37,20 +37,14 @@ def remove_tasks( # noqa: C901, PLR0912
failed_stash_revs.append(failed_rev_all[entry.stash_rev])
try:
for (
msg,
queue_entry,
) in celery_queue._iter_queued():
for msg, queue_entry in celery_queue._iter_queued():
if queue_entry.stash_rev in stash_revs and msg.delivery_tag:
celery_queue.celery.reject(msg.delivery_tag)
finally:
celery_queue.stash.remove_revs(list(stash_revs.values()))
try:
for (
msg,
queue_entry,
) in celery_queue._iter_processed():
for msg, queue_entry in celery_queue._iter_processed():
if queue_entry not in done_entry_set:
continue
task_id = msg.headers["id"]
@@ -115,10 +109,7 @@ def celery_clear(
return removed
def celery_remove(
self: "LocalCeleryQueue",
revs: Collection[str],
) -> list[str]:
def celery_remove(self: "LocalCeleryQueue", revs: Collection[str]) -> list[str]:
"""Remove the specified entries from the queue.
Arguments:
+1 -5
View File
@@ -23,11 +23,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]:
remote_url : remote executor's url
"""
refs = []
for ref in iter_remote_refs(
scm,
remote_url,
base=EXPS_NAMESPACE,
):
for ref in iter_remote_refs(scm, remote_url, base=EXPS_NAMESPACE):
if not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH:
refs.append(ref)
return refs
+1 -6
View File
@@ -182,12 +182,7 @@ class WorkspaceQueue(BaseStashQueue):
def shutdown(self, kill: bool = False):
raise NotImplementedError
def logs(
self,
rev: str,
encoding: Optional[str] = None,
follow: bool = False,
):
def logs(self, rev: str, encoding: Optional[str] = None, follow: bool = False):
raise NotImplementedError
def get_running_exp(self) -> Optional[str]:
+1 -8
View File
@@ -1,14 +1,7 @@
from collections import Counter, defaultdict
from collections.abc import Iterable, Iterator, Mapping
from datetime import date, datetime
from typing import (
TYPE_CHECKING,
Any,
Literal,
NamedTuple,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Optional, Union
from dvc.exceptions import InvalidArgumentError
from dvc.log import logger
+2 -10
View File
@@ -118,20 +118,12 @@ class ApplyStash(Stash):
m = self.MESSAGE_RE.match(msg)
if m:
revs[entry.new_sha.decode("utf-8")] = ApplyStashEntry(
i,
m.group("head_rev"),
m.group("rev"),
m.group("name"),
i, m.group("head_rev"), m.group("rev"), m.group("name")
)
return revs
@classmethod
def format_message(
cls,
head_rev: str,
rev: str,
name: Optional[str] = None,
) -> str:
def format_message(cls, head_rev: str, rev: str, name: Optional[str] = None) -> str:
return cls.MESSAGE_FORMAT.format(
head_rev=head_rev, rev=rev, name=name if name else ""
)
+2 -12
View File
@@ -4,12 +4,7 @@ import sys
from collections import defaultdict
from collections.abc import Generator, Iterable, Mapping
from functools import wraps
from typing import (
TYPE_CHECKING,
Callable,
Optional,
Union,
)
from typing import TYPE_CHECKING, Callable, Optional, Union
from dvc.exceptions import InvalidArgumentError
from dvc.repo.experiments.exceptions import AmbiguousExpRefInfo
@@ -282,12 +277,7 @@ def fetch_all_exps(scm: "Git", url: str, progress: Optional[Callable] = None, **
for ref in iter_remote_refs(scm, url, base=EXPS_NAMESPACE)
if not _ignore_ref(ref)
]
scm.fetch_refspecs(
url,
refspecs,
progress=progress,
**kwargs,
)
scm.fetch_refspecs(url, refspecs, progress=progress, **kwargs)
def gen_random_name():
+1 -5
View File
@@ -163,11 +163,7 @@ def fetch( # noqa: PLR0913
tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())),
)
with ui.progress(
desc="Collecting",
unit="entry",
leave=True,
) as pb:
with ui.progress(desc="Collecting", unit="entry", leave=True) as pb:
data = collect(
[idx.data["repo"] for idx in indexes.values()],
"remote",
+1 -6
View File
@@ -61,9 +61,4 @@ def get(
else:
fs = repo.dvcfs
fs_path = fs.from_os_path(path)
download(
fs,
fs_path,
os.path.abspath(out),
jobs=jobs,
)
download(fs, fs_path, os.path.abspath(out), jobs=jobs)
+2 -12
View File
@@ -4,14 +4,7 @@ from collections import defaultdict
from collections.abc import Iterable, Iterator
from functools import partial
from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
Callable,
NamedTuple,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
from funcy.debug import format_time
@@ -577,10 +570,7 @@ class Index:
return by_workspace
@staticmethod
def _hash_targets(
targets: Iterable[Optional[str]],
**kwargs: Any,
) -> int:
def _hash_targets(targets: Iterable[Optional[str]], **kwargs: Any) -> int:
return hash(
(
frozenset(targets),
+1 -4
View File
@@ -84,10 +84,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): # noqa:
proj = Repo(root_dir)
with proj.scm_context(autostage=True) as context:
files = [
config.files["repo"],
dvcignore,
]
files = [config.files["repo"], dvcignore]
ignore_file = context.scm.ignore_file
if ignore_file:
files.extend([os.path.join(dvc_dir, ignore_file)])
+1 -7
View File
@@ -1,13 +1,7 @@
import os
from collections.abc import Iterable, Iterator
from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
Optional,
TypedDict,
Union,
)
from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union
from funcy import ldistinct
from scmrepo.exceptions import SCMError
+1 -5
View File
@@ -18,11 +18,7 @@ logger = logger.getChild(__name__)
@map_scm_exception()
def _external_repo(
url,
rev: Optional[str] = None,
**kwargs,
) -> "Repo":
def _external_repo(url, rev: Optional[str] = None, **kwargs) -> "Repo":
logger.debug("Creating external repo %s@%s", url, rev)
path = _cached_clone(url, rev)
# Local HEAD points to the tip of whatever branch we first cloned from
+8 -37
View File
@@ -6,13 +6,7 @@ from collections.abc import Iterator
from copy import deepcopy
from functools import partial
from multiprocessing import cpu_count
from typing import (
TYPE_CHECKING,
Any,
Callable,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
import dpath
import dpath.options
@@ -415,10 +409,7 @@ def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = No
onerror=onerror,
)
dpath.merge(
result,
{"": unpacked},
)
dpath.merge(result, {"": unpacked})
return result
@@ -458,15 +449,9 @@ def _resolve_definitions(
data_path = _normpath(fs.join(config_dir, plot_id))
if _matches(targets, config_path, plot_id):
unpacked = unpack_if_dir(
fs,
data_path,
props={**plot_props, **props},
onerror=onerror,
)
dpath.merge(
result,
unpacked,
fs, data_path, props={**plot_props, **props}, onerror=onerror
)
dpath.merge(result, unpacked)
elif _matches(targets, config_path, plot_id):
adjusted_props = _adjust_sources(fs, plot_props, config_dir)
dpath.merge(result, {"data": {plot_id: {**adjusted_props, **props}}})
@@ -489,17 +474,9 @@ def _collect_pipeline_files(repo, targets: list[str], props, onerror=None):
dvcfile_defs_dict[k] = v
resolved = _resolve_definitions(
repo.dvcfs,
targets,
props,
dvcfile_path,
dvcfile_defs_dict,
onerror=onerror,
)
dpath.merge(
result,
{dvcfile_path: resolved},
repo.dvcfs, targets, props, dvcfile_path, dvcfile_defs_dict, onerror=onerror
)
dpath.merge(result, {dvcfile_path: resolved})
return result
@@ -515,15 +492,9 @@ def _collect_definitions(
props = props or {}
fs = repo.dvcfs
dpath.merge(
result,
_collect_pipeline_files(repo, targets, props, onerror=onerror),
)
dpath.merge(result, _collect_pipeline_files(repo, targets, props, onerror=onerror))
dpath.merge(
result,
_collect_output_plots(repo, targets, props, onerror=onerror),
)
dpath.merge(result, _collect_output_plots(repo, targets, props, onerror=onerror))
for target in targets:
if not result or fs.exists(target):
+1 -5
View File
@@ -120,11 +120,7 @@ def push( # noqa: PLR0913
tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())),
)
with ui.progress(
desc="Collecting",
unit="entry",
leave=True,
) as pb:
with ui.progress(desc="Collecting", unit="entry", leave=True) as pb:
data = collect(
[idx.data["repo"] for idx in indexes.values()],
"remote",
+1 -9
View File
@@ -1,13 +1,5 @@
from collections.abc import Iterable
from typing import (
TYPE_CHECKING,
Callable,
NoReturn,
Optional,
TypeVar,
Union,
cast,
)
from typing import TYPE_CHECKING, Callable, NoReturn, Optional, TypeVar, Union, cast
from funcy import ldistinct
+1 -6
View File
@@ -3,12 +3,7 @@ import shlex
from collections.abc import Iterable, Iterator
from contextlib import contextmanager
from functools import wraps
from typing import (
TYPE_CHECKING,
Any,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, Optional, Union
from dvc.log import logger
from dvc.utils import relpath
+1 -6
View File
@@ -402,12 +402,7 @@ class StageLoad:
from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound
try:
stages = self.collect(
target,
with_deps,
recursive,
graph,
)
stages = self.collect(target, with_deps, recursive, graph)
except StageFileDoesNotExistError as exc:
# collect() might try to use `target` as a stage name
# and throw error that dvc.yaml does not exist, whereas it
+1 -4
View File
@@ -61,10 +61,7 @@ def update( # noqa: C901
)
if to_remote:
raise InvalidArgumentError("--to-remote can't be used with worktree update")
update_worktree_stages(
self,
other_stage_infos,
)
update_worktree_stages(self, other_stage_infos)
stages = import_stages | {stage_info.stage for stage_info in other_stage_infos}
return list(stages)
+7 -32
View File
@@ -162,20 +162,13 @@ def _merge_push_meta( # noqa: C901
out.meta.remote = remote
def update_worktree_stages(
repo: "Repo",
stage_infos: Iterable["StageInfo"],
):
def update_worktree_stages(repo: "Repo", stage_infos: Iterable["StageInfo"]):
from dvc.repo.index import IndexView
def outs_filter(out: "Output") -> bool:
return out.is_in_repo and out.use_cache and out.can_push
view = IndexView(
repo.index,
stage_infos,
outs_filter=outs_filter,
)
view = IndexView(repo.index, stage_infos, outs_filter=outs_filter)
local_index = view.data["repo"]
remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {}
for stage in view.stages:
@@ -194,10 +187,7 @@ def _update_worktree_out(
remote_name = out.remote or out.meta.remote
if not remote_name:
logger.warning(
"Could not update '%s', it was never pushed to a remote",
out,
)
logger.warning("Could not update '%s', it was never pushed to a remote", out)
return
if remote_name in remote_indexes:
@@ -211,10 +201,7 @@ def _update_worktree_out(
remote_indexes[remote_name] = remote, remote_index
_workspace, key = out.index_key
if key not in remote_index:
logger.warning(
"Could not update '%s', it does not exist in the remote",
out,
)
logger.warning("Could not update '%s', it does not exist in the remote", out)
return
entry = remote_index[key]
@@ -226,20 +213,11 @@ def _update_worktree_out(
for subkey, subentry in remote_index.iteritems(key)
)
):
logger.warning(
"Could not update '%s', directory is empty in the remote",
out,
)
logger.warning("Could not update '%s', directory is empty in the remote", out)
return
_fetch_out_changes(out, local_index, remote_index, remote)
_update_out_meta(
repo,
out,
local_index,
remote_index,
remote,
)
_update_out_meta(repo, out, local_index, remote_index, remote)
def _fetch_out_changes(
@@ -253,10 +231,7 @@ def _fetch_out_changes(
old, new = _get_diff_indexes(out, local_index, remote_index)
with TqdmCallback(
unit="entry",
desc="Comparing indexes",
) as cb:
with TqdmCallback(unit="entry", desc="Comparing indexes") as cb:
diff = compare(
old,
new,
+1 -7
View File
@@ -3,13 +3,7 @@ import os
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from functools import partial
from typing import (
TYPE_CHECKING,
Literal,
Optional,
Union,
overload,
)
from typing import TYPE_CHECKING, Literal, Optional, Union, overload
from funcy import group_by
from scmrepo.base import Base # noqa: F401
+2 -12
View File
@@ -4,13 +4,7 @@ from collections import defaultdict
from collections.abc import Iterable
from contextlib import suppress
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Any,
Optional,
TypeVar,
Union,
)
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
from funcy import project
@@ -664,11 +658,7 @@ class Stage(params.StageParams):
return stats
for out in self.filter_outs(kwargs.get("filter_info")):
key, outs = self._checkout(
out,
allow_missing=allow_missing,
**kwargs,
)
key, outs = self._checkout(out, allow_missing=allow_missing, **kwargs)
if key:
stats[key].extend(outs)
return stats
+1 -4
View File
@@ -266,10 +266,7 @@ class StageCache:
src_name = from_fs.name(src)
parent_name = from_fs.name(from_fs.parent(src))
with TqdmCallback(
desc=src_name,
bytes=True,
) as cb:
with TqdmCallback(desc=src_name, bytes=True) as cb:
func(from_fs, src, to_fs, dst, callback=cb)
ret.append((parent_name, src_name))
return ret
+2 -11
View File
@@ -44,13 +44,7 @@ def update_import(
stage.frozen = frozen
def sync_import(
stage,
dry=False,
force=False,
jobs=None,
no_download=False,
):
def sync_import(stage, dry=False, force=False, jobs=None, no_download=False):
"""Synchronize import's outs to the workspace."""
logger.info("Importing '%s' -> '%s'", stage.deps[0], stage.outs[0])
if dry:
@@ -64,7 +58,4 @@ def sync_import(
if stage.is_repo_import or stage.is_db_import:
stage.deps[0].update()
else:
stage.deps[0].download(
stage.outs[0],
jobs=jobs,
)
stage.deps[0].download(stage.outs[0], jobs=jobs)
+2 -9
View File
@@ -25,12 +25,7 @@ logger = logger.getChild(__name__)
class StageLoader(Mapping):
def __init__(
self,
dvcfile: "ProjectFile",
data,
lockfile_data=None,
):
def __init__(self, dvcfile: "ProjectFile", data, lockfile_data=None):
self.dvcfile = dvcfile
self.resolver = self.dvcfile.resolver
self.data = data or {}
@@ -190,9 +185,7 @@ class SingleStageLoader(Mapping):
def __getitem__(self, item):
if item:
logger.warning(
"Ignoring name '%s' for single stage in '%s'.",
item,
self.dvcfile,
"Ignoring name '%s' for single stage in '%s'.", item, self.dvcfile
)
# during `load`, we remove attributes from stage data, so as to
# not duplicate, therefore, for MappingView, we need to deepcopy.
+1 -7
View File
@@ -119,13 +119,7 @@ def _pull_missing_deps(stage):
stage.repo.pull(dep.def_path)
def run_stage(
stage,
dry=False,
force=False,
run_env=None,
**kwargs,
):
def run_stage(stage, dry=False, force=False, run_env=None, **kwargs):
if not force:
if kwargs.get("pull") and not dry:
_pull_missing_deps(stage)
+1 -7
View File
@@ -1,13 +1,7 @@
from collections import OrderedDict
from collections.abc import Iterable
from operator import attrgetter
from typing import (
TYPE_CHECKING,
Any,
Optional,
Union,
no_type_check,
)
from typing import TYPE_CHECKING, Any, Optional, Union, no_type_check
from funcy import post_processing
+1 -6
View File
@@ -15,12 +15,7 @@ class TestAPI:
assert api.get_url("foo") == expected_url
def test_open(self, tmp_dir, dvc, remote):
tmp_dir.dvc_gen(
{
"foo": "foo-text",
"dir": {"bar": "bar-text"},
}
)
tmp_dir.dvc_gen({"foo": "foo-text", "dir": {"bar": "bar-text"}})
dvc.push()
# Remove cache to force download
@@ -5,13 +5,7 @@ import pytest
pytestmark = pytest.mark.requires(minversion=(2, 15, 0), reason="new command")
def test_data_status(
bench_dvc,
tmp_dir,
scm,
dvc,
make_dataset,
):
def test_data_status(bench_dvc, tmp_dir, scm, dvc, make_dataset):
args = ("data", "status")
dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
rmtree(dvc.tmp_dir)
+1 -5
View File
@@ -86,11 +86,7 @@ def pytest_addoption(parser):
help="Path or url to dvc-bench git repo (for loading benchmark datasets)",
)
parser.addoption(
"--project-rev",
type=str,
help="Project revision to test",
)
parser.addoption("--project-rev", type=str, help="Project revision to test")
parser.addoption(
"--project-git-repo",
+1 -5
View File
@@ -246,11 +246,7 @@ def docker_services(
try:
cmd = "docker-compose version"
subprocess.check_output(
cmd,
stderr=subprocess.STDOUT,
shell=True, # noqa: S602
)
subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) # noqa: S602
except subprocess.CalledProcessError as err:
out = (err.output or b"").decode("utf-8")
pytest.skip(f"docker-compose is not installed: {out}")
+1 -3
View File
@@ -209,9 +209,7 @@ class URLInfo(_BasePath):
return self._spath
@cached_property
def _path(
self,
) -> "_URLPathInfo":
def _path(self) -> "_URLPathInfo":
return _URLPathInfo(self._spath)
@property
+1 -3
View File
@@ -20,7 +20,5 @@ def _add_script(tmp_dir, path, contents=""):
@pytest.fixture
def copy_script(
tmp_dir,
):
def copy_script(tmp_dir):
return _add_script(tmp_dir, "copy.py", COPY_SCRIPT)
+2 -8
View File
@@ -173,9 +173,7 @@ class TestImportURLVersionAware:
dvc.cache.local.clear()
remove(tmp_dir / "data_dir")
assert dvc.pull()["fetched"] == 1
assert (tmp_dir / "data_dir").read_text() == {
"subdir": {"file": "file"},
}
assert (tmp_dir / "data_dir").read_text() == {"subdir": {"file": "file"}}
dvc.commit(force=True)
assert dvc.status() == {}
@@ -193,11 +191,7 @@ class TestLsUrl:
cloud.gen({fname: "foo contents"})
fs, fs_path = parse_external_url(cloud.url, cloud.config)
result = ls_url(str(cloud / fname), fs_config=cloud.config)
match_files(
fs,
result,
[{"path": fs.join(fs_path, fname), "isdir": False}],
)
match_files(fs, result, [{"path": fs.join(fs_path, fname), "isdir": False}])
def test_dir(self, cloud):
cloud.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"})
+1 -8
View File
@@ -1,13 +1,6 @@
from collections.abc import Iterable, Iterator, Sequence
from contextlib import contextmanager, nullcontext
from typing import (
TYPE_CHECKING,
Any,
Callable,
Optional,
TextIO,
Union,
)
from typing import TYPE_CHECKING, Any, Callable, Optional, TextIO, Union
import colorama
+1 -3
View File
@@ -17,9 +17,7 @@ def parse_params(path_params: Iterable[str]) -> list[dict[str, list[str]]]:
return [{path: params} for path, params in ret.items()]
def to_path_overrides(
path_params: Iterable[str],
) -> dict[str, list[str]]:
def to_path_overrides(path_params: Iterable[str]) -> dict[str, list[str]]:
"""Group overrides by path"""
from dvc.dependency.param import ParamsDependency
+1 -9
View File
@@ -1,15 +1,7 @@
"""Common utilities for serialize."""
import os
from contextlib import AbstractContextManager, contextmanager
from typing import (
TYPE_CHECKING,
Any,
Callable,
Optional,
Protocol,
TextIO,
Union,
)
from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol, TextIO, Union
from funcy import reraise
+9 -45
View File
@@ -31,38 +31,15 @@ def params_repo(tmp_dir, scm, dvc):
tmp_dir.gen("params.json", '{"bar": 2, "foobar": 3}')
tmp_dir.gen("other_params.json", '{"foo": {"bar": 4}}')
dvc.run(
name="stage-0",
cmd="echo stage-0",
)
dvc.run(name="stage-0", cmd="echo stage-0")
dvc.run(
name="stage-1",
cmd="echo stage-1",
params=["foo", "params.json:bar"],
)
dvc.run(name="stage-1", cmd="echo stage-1", params=["foo", "params.json:bar"])
dvc.run(
name="stage-2",
cmd="echo stage-2",
params=["other_params.json:foo"],
)
dvc.run(name="stage-2", cmd="echo stage-2", params=["other_params.json:foo"])
dvc.run(
name="stage-3",
cmd="echo stage-2",
params=["params.json:foobar"],
)
dvc.run(name="stage-3", cmd="echo stage-2", params=["params.json:foobar"])
scm.add(
[
"params.yaml",
"params.json",
"other_params.json",
"dvc.yaml",
"dvc.lock",
]
)
scm.add(["params.yaml", "params.json", "other_params.json", "dvc.yaml", "dvc.lock"])
scm.commit("commit dvc files")
tmp_dir.gen("params.yaml", "foo: 5")
@@ -77,10 +54,7 @@ def metrics_repo(tmp_dir, scm, dvc, run_copy_metrics):
scm.commit("prepare data")
sub_dir = tmp_dir / "eval"
sub_dir.mkdir()
tmp_dir.gen(
"tmp_train_val_metrics.json",
json.dumps(TRAIN_METRICS[0]),
)
tmp_dir.gen("tmp_train_val_metrics.json", json.dumps(TRAIN_METRICS[0]))
train_metrics_file = os.path.join(sub_dir, "train_val_metrics.json")
run_copy_metrics(
"tmp_train_val_metrics.json",
@@ -107,10 +81,7 @@ def metrics_repo(tmp_dir, scm, dvc, run_copy_metrics):
scm.commit("test model")
with tmp_dir.branch("better-model", new=True):
tmp_dir.gen(
"tmp_train_val_metrics.json",
json.dumps(TRAIN_METRICS[1]),
)
tmp_dir.gen("tmp_train_val_metrics.json", json.dumps(TRAIN_METRICS[1]))
run_copy_metrics(
"tmp_train_val_metrics.json",
train_metrics_file,
@@ -243,11 +214,7 @@ def test_params_show_while_running_stage(tmp_dir, dvc):
def test_params_show_repo(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.scm_gen("params.yaml", "foo: 1", commit="Create params.yaml")
erepo_dir.dvc.run(
name="stage-1",
cmd="echo stage-1",
params=["foo"],
)
erepo_dir.dvc.run(name="stage-1", cmd="echo stage-1", params=["foo"])
assert api.params_show(repo=erepo_dir) == {"foo": 1}
@@ -268,10 +235,7 @@ def test_params_show_no_params_found(tmp_dir, dvc):
def test_params_show_stage_without_params(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "foo: 1")
dvc.run(
name="stage-0",
cmd="echo stage-0",
)
dvc.run(name="stage-0", cmd="echo stage-0")
assert api.params_show(stages="stage-0") == {}
+3 -18
View File
@@ -70,9 +70,7 @@ def test_artifacts_add_subdir(tmp_dir, dvc):
name: Artifact(**values) for name, values in dvcyaml["artifacts"].items()
}
artifacts["new"] = new_art
assert tmp_dir.dvc.artifacts.read() == {
f"subdir{os.path.sep}dvc.yaml": artifacts,
}
assert tmp_dir.dvc.artifacts.read() == {f"subdir{os.path.sep}dvc.yaml": artifacts}
def test_artifacts_add_abspath(tmp_dir, dvc):
@@ -114,11 +112,7 @@ bad_dvcyaml_extra_field = {
}
bad_dvcyaml_missing_path = {
"artifacts": {
"lol": {},
}
}
bad_dvcyaml_missing_path = {"artifacts": {"lol": {}}}
@pytest.mark.parametrize(
@@ -148,16 +142,7 @@ def test_artifacts_read_fails_on_id_duplication(tmp_dir, dvc):
@pytest.mark.parametrize(
"name",
[
"1",
"m",
"nn",
"m1",
"1nn",
"model-prod",
"model-prod-v1",
],
"name", ["1", "m", "nn", "m1", "1nn", "model-prod", "model-prod-v1"]
)
def test_name_is_compatible(name):
check_name_format(name)
+1 -5
View File
@@ -34,10 +34,6 @@ def params_repo(tmp_dir, scm, dvc):
(tmp_dir / "params.yaml").dump(
{"foo": [{"bar": 1}, {"baz": 2}], "goo": {"bag": 3.0}, "lorem": False}
)
dvc.run(
cmd="echo foo",
params=["params.yaml:"],
name="foo",
)
dvc.run(cmd="echo foo", params=["params.yaml:"], name="foo")
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml"])
scm.commit("init")
+1 -5
View File
@@ -46,11 +46,7 @@ def test_apply_failed(tmp_dir, scm, dvc, failed_exp_stage, mocker):
),
],
)
mocker.patch.object(
queue,
"iter_queued",
return_value=[],
)
mocker.patch.object(queue, "iter_queued", return_value=[])
dvc.experiments.apply(exp_rev)
assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 3"
+8 -41
View File
@@ -100,14 +100,7 @@ def test_file_permissions(tmp_dir, scm, dvc, exp_stage, mocker):
assert stat.S_IMODE(os.stat(tmp_dir / "copy.py").st_mode) == mode
def test_failed_exp_workspace(
tmp_dir,
scm,
dvc,
failed_exp_stage,
mocker,
capsys,
):
def test_failed_exp_workspace(tmp_dir, scm, dvc, failed_exp_stage, mocker, capsys):
tmp_dir.gen("params.yaml", "foo: 2")
with pytest.raises(ReproductionError):
dvc.experiments.run(failed_exp_stage.addressing)
@@ -445,13 +438,7 @@ def test_subrepo(tmp_dir, request, scm, workspace):
name="copy-file",
no_exec=True,
)
scm.add(
[
subrepo / "dvc.yaml",
subrepo / "copy.py",
subrepo / "params.yaml",
]
)
scm.add([subrepo / "dvc.yaml", subrepo / "copy.py", subrepo / "params.yaml"])
scm.commit("init")
results = subrepo.dvc.experiments.run(
@@ -593,11 +580,7 @@ def test_experiment_name_invalid(tmp_dir, scm, dvc, exp_stage, mocker):
new_mock = mocker.spy(BaseStashQueue, "_stash_exp")
with pytest.raises(InvalidArgumentError):
dvc.experiments.run(
exp_stage.addressing,
name="fo^o",
params=["foo=3"],
)
dvc.experiments.run(exp_stage.addressing, name="fo^o", params=["foo=3"])
new_mock.assert_not_called()
@@ -651,10 +634,7 @@ def test_run_env(tmp_dir, dvc, scm, mocker):
)
(tmp_dir / "dump_run_env.py").write_text(dump_run_env)
baseline = scm.get_rev()
dvc.stage.add(
cmd="python dump_run_env.py",
name="run_env",
)
dvc.stage.add(cmd="python dump_run_env.py", name="run_env")
dvc.experiments.run()
assert (tmp_dir / DVC_EXP_BASELINE_REV).read_text().strip() == baseline
assert (tmp_dir / DVC_EXP_NAME).read_text().strip()
@@ -728,11 +708,7 @@ def test_local_config_is_propagated_to_tmp(tmp_dir, scm, dvc):
@pytest.mark.parametrize("tmp", [True, False])
def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
(tmp_dir / "dvc.yaml").dump(
{
"metrics": ["metrics.json"],
"params": ["params.yaml"],
"plots": ["plots.csv"],
}
{"metrics": ["metrics.json"], "params": ["params.yaml"], "plots": ["plots.csv"]}
)
stage = dvc.stage.add(
cmd="touch metrics.json && touch params.yaml && touch plots.csv",
@@ -748,10 +724,7 @@ def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
@pytest.mark.parametrize("tmp", [True, False])
def test_copy_paths(tmp_dir, scm, dvc, tmp):
stage = dvc.stage.add(
cmd="cat file && ls dir",
name="foo",
)
stage = dvc.stage.add(cmd="cat file && ls dir", name="foo")
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
(tmp_dir / "dir").mkdir()
@@ -770,10 +743,7 @@ def test_copy_paths(tmp_dir, scm, dvc, tmp):
def test_copy_paths_errors(tmp_dir, scm, dvc, mocker):
stage = dvc.stage.add(
cmd="echo foo",
name="foo",
)
stage = dvc.stage.add(cmd="echo foo", name="foo")
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
with pytest.raises(DvcException, match="Unable to copy"):
@@ -814,10 +784,7 @@ def test_mixed_git_dvc_out(tmp_dir, scm, dvc, exp_stage):
@pytest.mark.parametrize("tmp", [True, False])
def test_custom_commit_message(tmp_dir, scm, dvc, tmp):
stage = dvc.stage.add(
cmd="echo foo",
name="foo",
)
stage = dvc.stage.add(cmd="echo foo", name="foo")
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
exp = first(
+4 -24
View File
@@ -10,15 +10,7 @@ def to_dict(tasks):
@pytest.mark.parametrize("follow", [True, False])
def test_celery_logs(
tmp_dir,
scm,
dvc,
failed_exp_stage,
follow,
capsys,
test_queue,
):
def test_celery_logs(tmp_dir, scm, dvc, failed_exp_stage, follow, capsys, test_queue):
celery_queue = dvc.experiments.celery_queue
dvc.experiments.run(failed_exp_stage.addressing, queue=True, name="foo")
dvc.experiments.run(run_all=True)
@@ -37,23 +29,14 @@ def test_queue_doesnt_remove_untracked_params_file(tmp_dir, dvc, scm):
"""Regression test for https://github.com/iterative/dvc/issues/7842"""
tmp_dir.gen("params.yaml", "foo: 1")
stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
scm.add(
[
"dvc.yaml",
"dvc.lock",
".gitignore",
]
)
scm.add(["dvc.yaml", "dvc.lock", ".gitignore"])
scm.commit("init")
dvc.experiments.run(stage.addressing, params=["foo=2"], queue=True)
assert (tmp_dir / "params.yaml").exists()
def test_copy_paths_queue(tmp_dir, scm, dvc):
stage = dvc.stage.add(
cmd="cat file && ls dir",
name="foo",
)
stage = dvc.stage.add(cmd="cat file && ls dir", name="foo")
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
(tmp_dir / "dir").mkdir()
@@ -72,10 +55,7 @@ def test_copy_paths_queue(tmp_dir, scm, dvc):
def test_custom_commit_message_queue(tmp_dir, scm, dvc):
stage = dvc.stage.add(
cmd="echo foo",
name="foo",
)
stage = dvc.stage.add(cmd="echo foo", name="foo")
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
dvc.experiments.run(stage.addressing, queue=True, message="custom commit message")
+1 -5
View File
@@ -356,11 +356,7 @@ def test_get(tmp_dir, scm, dvc, exp_stage, erepo_dir, use_ref):
exp_ref = first(exp_refs_by_rev(scm, exp_rev))
with erepo_dir.chdir():
Repo.get(
str(tmp_dir),
"params.yaml",
rev=exp_ref.name if use_ref else exp_rev,
)
Repo.get(str(tmp_dir), "params.yaml", rev=exp_ref.name if use_ref else exp_rev)
assert (erepo_dir / "params.yaml").read_text().strip() == "foo: 2"
+2 -9
View File
@@ -121,11 +121,7 @@ def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
(tmp_dir / "dvc.yaml").dump(
{
"metrics": ["metrics.json"],
"params": ["params.yaml"],
"plots": ["plots.csv"],
}
{"metrics": ["metrics.json"], "params": ["params.yaml"], "plots": ["plots.csv"]}
)
stage = dvc.stage.add(
cmd="touch metrics.json && touch params.yaml && touch plots.csv",
@@ -140,10 +136,7 @@ def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
def test_untracked_dvclock_is_included_in_exp(tmp_dir, scm, dvc):
stage = dvc.stage.add(
cmd="echo foo",
name="foo",
)
stage = dvc.stage.add(cmd="echo foo", name="foo")
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
dvc.reproduce(stage.addressing)
+1 -5
View File
@@ -103,11 +103,7 @@ def test_hydra_sweep(
patched = mocker.patch.object(dvc.experiments, "queue_one")
if hydra_enabled:
hydra_setup(
tmp_dir,
config_dir="conf",
config_name="config",
)
hydra_setup(tmp_dir, config_dir="conf", config_name="config")
with dvc.config.edit() as conf:
conf["hydra"]["enabled"] = True
+9 -47
View File
@@ -49,11 +49,7 @@ def make_executor(local=None, **kwargs):
local_executor.update(local)
else:
local_executor = ANY
data = {
"state": ANY,
"local": local_executor,
"name": ANY,
}
data = {"state": ANY, "local": local_executor, "name": ANY}
data.update(kwargs)
return data
@@ -62,13 +58,7 @@ def make_data(params=None, **kwargs):
params = {"data": params or {"foo": 1}}
data = {
"rev": ANY,
"deps": {
"copy.py": {
"hash": ANY,
"size": ANY,
"nfiles": None,
}
},
"deps": {"copy.py": {"hash": ANY, "size": ANY, "nfiles": None}},
"metrics": {"metrics.yaml": params},
"outs": {},
"params": {"params.yaml": params},
@@ -233,23 +223,14 @@ def test_show_failed_experiment(tmp_dir, scm, dvc, failed_exp_stage, test_queue)
"experiments": None,
}
],
"executor": make_executor(
state="failed",
local={"returncode": 255},
),
"executor": make_executor(state="failed", local={"returncode": 255}),
"name": ANY,
}
],
}
def test_show_filter(
tmp_dir,
scm,
dvc,
capsys,
copy_script,
):
def test_show_filter(tmp_dir, scm, dvc, capsys, copy_script):
capsys.readouterr()
params_file = tmp_dir / "params.yaml"
@@ -487,10 +468,7 @@ def test_show_csv(tmp_dir, scm, dvc, exp_stage, capsys):
def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script):
params_file = tmp_dir / "params.yaml"
params_data = {
"foo": 1,
"goobar": 1,
}
params_data = {"foo": 1, "goobar": 1}
(tmp_dir / params_file).dump(params_data)
dvc.run(
@@ -534,10 +512,7 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script):
@pytest.mark.vscode
def test_show_outs(tmp_dir, dvc, scm, erepo_dir, copy_script):
params_file = tmp_dir / "params.yaml"
params_data = {
"foo": 1,
"bar": 1,
}
params_data = {"foo": 1, "bar": 1}
(tmp_dir / params_file).dump(params_data)
dvc.run(
@@ -641,9 +616,7 @@ def test_show_outs(tmp_dir, dvc, scm, erepo_dir, copy_script):
def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script):
params_file = tmp_dir / "params.yaml"
params_data = {
"foo": 1,
}
params_data = {"foo": 1}
(tmp_dir / params_file).dump(params_data)
dvc.run(
@@ -674,14 +647,7 @@ def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script):
name="copy-file",
deps=["copy.py"],
)
scm.add(
[
"dvc.yaml",
"dvc.lock",
"params.yaml",
"scores.yaml",
]
)
scm.add(["dvc.yaml", "dvc.lock", "params.yaml", "scores.yaml"])
scm.commit("scores.yaml")
scores_rev = scm.get_rev()
@@ -709,11 +675,7 @@ def test_show_sorted_deps(tmp_dir, dvc, scm, capsys):
tmp_dir.gen("c", "c")
tmp_dir.gen("z", "z")
dvc.run(
cmd="echo foo",
name="deps",
deps=["a", "b", "z", "c"],
)
dvc.run(cmd="echo foo", name="deps", deps=["a", "b", "z", "c"])
capsys.readouterr()
assert main(["exp", "show", "--csv"]) == 0
+3 -13
View File
@@ -9,11 +9,7 @@ from dvc.exceptions import ReproductionError
@pytest.mark.parametrize("staged", [True, False])
def test_deleted(tmp_dir, scm, dvc, tmp, staged):
tmp_dir.scm_gen("file", "file", commit="commit file")
stage = dvc.stage.add(
cmd="cat file",
deps=["file"],
name="foo",
)
stage = dvc.stage.add(cmd="cat file", deps=["file"], name="foo")
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
file = tmp_dir / "file"
@@ -33,10 +29,7 @@ def test_deleted(tmp_dir, scm, dvc, tmp, staged):
@pytest.mark.parametrize("staged", [True, False])
def test_modified(tmp_dir, scm, dvc, caplog, tmp, staged):
tmp_dir.scm_gen("file", "file", commit="commit file")
stage = dvc.stage.add(
cmd="cat file",
name="foo",
)
stage = dvc.stage.add(cmd="cat file", name="foo")
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
(tmp_dir / "file").write_text("modified_file")
@@ -52,10 +45,7 @@ def test_modified(tmp_dir, scm, dvc, caplog, tmp, staged):
@pytest.mark.parametrize("tmp", [True, False])
def test_staged_new_file(tmp_dir, scm, dvc, tmp):
stage = dvc.stage.add(
cmd="cat file",
name="foo",
)
stage = dvc.stage.add(cmd="cat file", name="foo")
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
(tmp_dir / "file").write_text("file")
+1 -10
View File
@@ -3,16 +3,7 @@ from funcy import first
def test_generate_random_exp_name(tmp_dir, dvc, scm, exp_stage, mocker):
mocked_generator = mocker.MagicMock()
mocked_generator.choice.side_effect = [
0,
0,
0,
0,
1,
1,
0,
0,
]
mocked_generator.choice.side_effect = [0, 0, 0, 0, 1, 1, 0, 0]
mocker.patch(
"dvc.repo.experiments.utils.random.Random", return_value=mocked_generator
)
+2 -11
View File
@@ -160,13 +160,7 @@ def test_missing_cache(tmp_dir, dvc, run_copy_metrics):
result = dvc.metrics.show()
metrics2 = result[""]["data"].pop("metrics2.yaml")
assert isinstance(metrics2["error"], FileNotFoundError)
assert result == {
"": {
"data": {
"metrics.yaml": {"data": 1.1},
}
}
}
assert result == {"": {"data": {"metrics.yaml": {"data": 1.1}}}}
@pytest.mark.parametrize("use_dvc", [True, False])
@@ -276,10 +270,7 @@ def test_metrics_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
)
with (tmp_dir / "dvc.yaml").modify() as d:
# trying to make an output overlaps error
d["stages"]["corrupted-stage"] = {
"cmd": "mkdir data",
"outs": ["data"],
}
d["stages"]["corrupted-stage"] = {"cmd": "mkdir data", "outs": ["data"]}
# running by clearing and not clearing stuffs
# so as it works even for optimized cases
+1 -5
View File
@@ -238,11 +238,7 @@ def test_diff_targeted(tmp_dir, scm, dvc, run_copy):
def test_diff_without_targets_specified(tmp_dir, dvc, scm, file):
params_file = tmp_dir / file
params_file.dump({"foo": {"bar": "bar"}, "x": "0"})
dvc.stage.add(
name="test",
cmd=f"echo {file}",
params=[{file: None}],
)
dvc.stage.add(name="test", cmd=f"echo {file}", params=[{file: None}])
scm.add_commit([params_file, "dvc.yaml"], message="foo")
params_file.dump({"foo": {"bar": "baz"}, "y": "100"})
+1 -5
View File
@@ -123,11 +123,7 @@ def test_show_without_targets_specified(tmp_dir, dvc, scm, file):
params_file = tmp_dir / file
data = {"foo": {"bar": "bar"}, "x": "0"}
params_file.dump(data)
dvc.stage.add(
name="test",
cmd=f"echo {file}",
params=[{file: None}],
)
dvc.stage.add(name="test", cmd=f"echo {file}", params=[{file: None}])
assert dvc.params.show() == {"": {"data": {file: {"data": data}}}}
+1 -3
View File
@@ -71,9 +71,7 @@ def test_load_vars_with_relpath(tmp_dir, scm, dvc):
revisions = ["HEAD", "workspace"]
for rev in dvc.brancher(revs=["HEAD"]):
assert rev == revisions.pop()
d = {
"vars": [f"../{DEFAULT_PARAMS_FILE}"],
}
d = {"vars": [f"../{DEFAULT_PARAMS_FILE}"]}
resolver = DataResolver(dvc, "subdir", d)
assert resolver.context == deepcopy(DATA)
+2 -10
View File
@@ -155,10 +155,7 @@ def test_plots_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
)
with (tmp_dir / "dvc.yaml").modify() as d:
# trying to make an output overlaps error
d["stages"]["corrupted-stage"] = {
"cmd": "mkdir data",
"outs": ["data"],
}
d["stages"]["corrupted-stage"] = {"cmd": "mkdir data", "outs": ["data"]}
# running by clearing and not clearing stuffs
# so as it works even for optimized cases
@@ -356,12 +353,7 @@ def test_collect_non_existing_dir(tmp_dir, dvc, run_copy_metrics):
),
],
)
def test_top_level_plots(
tmp_dir,
dvc,
plot_config,
expected_datafiles,
):
def test_top_level_plots(tmp_dir, dvc, plot_config, expected_datafiles):
data = {
"data1.json": [
{"a": 1, "b": 0.1, "c": 0.01},
+25 -121
View File
@@ -364,10 +364,7 @@ def test_repro_multiple_params(tmp_dir, dvc):
name="read_params",
deps=["foo"],
outs=["bar"],
params=[
"params2.yaml:lists,floats,name",
"answer,floats,nested.nested1",
],
params=["params2.yaml:lists,floats,name", "answer,floats,nested.nested1"],
cmd="cat params2.yaml params.yaml > bar",
)
@@ -528,11 +525,7 @@ def test_repro_skip_pull_if_no_run_cache_is_passed(tmp_dir, dvc, mocker, local_r
assert not spy_pull.called
def test_repro_no_commit(
tmp_dir,
dvc,
copy_script,
):
def test_repro_no_commit(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
@@ -548,16 +541,9 @@ def test_repro_no_commit(
assert not os.path.isdir(dvc.cache.local.path)
def test_repro_all_pipelines(
mocker,
dvc,
):
def test_repro_all_pipelines(mocker, dvc):
stages = [
dvc.run(
outs=["start.txt"],
cmd="echo start > start.txt",
name="start",
),
dvc.run(outs=["start.txt"], cmd="echo start > start.txt", name="start"),
dvc.run(
deps=["start.txt"],
outs=["middle.txt"],
@@ -588,10 +574,7 @@ def test_repro_all_pipelines(
class TestReproAlreadyCached:
def test(
self,
dvc,
):
def test(self, dvc):
stage = dvc.run(
always_changed=True,
deps=[],
@@ -604,11 +587,7 @@ class TestReproAlreadyCached:
assert run_out.hash_info != repro_out.hash_info
def test_force_with_dependencies(
self,
tmp_dir,
dvc,
):
def test_force_with_dependencies(self, tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
name="datetime",
@@ -654,11 +633,7 @@ def test_repro_shell(tmp_dir, monkeypatch, dvc):
assert (tmp_dir / "shell.txt").read_text().rstrip() == shell
def test_cmd_repro(
tmp_dir,
dvc,
copy_script,
):
def test_cmd_repro(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
@@ -679,11 +654,7 @@ def test_cmd_repro(
assert ret != 0
def test_repro_dep_under_dir(
tmp_dir,
dvc,
copy_script,
):
def test_repro_dep_under_dir(tmp_dir, dvc, copy_script):
tmp_dir.gen("foo", "foo")
tmp_dir.dvc_gen("data", {"file": "file", "sub": {"foo": "foo"}})
@@ -741,11 +712,7 @@ def test_repro_force(tmp_dir, dvc, copy_script):
assert len(stages) == 2
def test_repro_changed_code(
tmp_dir,
dvc,
copy_script,
):
def test_repro_changed_code(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
@@ -762,11 +729,7 @@ def test_repro_changed_code(
assert len(stages) == 1
def test_repro_changed_data(
tmp_dir,
dvc,
copy_script,
):
def test_repro_changed_data(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
@@ -783,11 +746,7 @@ def test_repro_changed_data(
assert len(stages) == 2
def test_repro_dry(
tmp_dir,
dvc,
copy_script,
):
def test_repro_dry(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
@@ -808,11 +767,7 @@ def test_repro_dry(
assert not filecmp.cmp("file1", "bar", shallow=False)
def test_repro_up_to_date(
tmp_dir,
dvc,
copy_script,
):
def test_repro_up_to_date(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
@@ -871,11 +826,7 @@ def test_repro_dry_no_exec(tmp_dir, dvc):
assert ret == 0
def test_repro_changed_deep_data(
tmp_dir,
dvc,
copy_script,
):
def test_repro_changed_deep_data(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
dvc.run(
@@ -955,11 +906,7 @@ def test_repro_force_downstream_do_not_force_independent_stages(tmp_dir, dvc, ru
assert dvc.reproduce(force_downstream=True) == [foo1, foo2, cat]
def test_repro_pipeline(
tmp_dir,
dvc,
copy_script,
):
def test_repro_pipeline(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
dvc.run(
@@ -978,11 +925,7 @@ def test_repro_pipeline(
assert len(stages) == 3
def test_repro_pipeline_cli(
tmp_dir,
dvc,
copy_script,
):
def test_repro_pipeline_cli(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
@@ -995,11 +938,7 @@ def test_repro_pipeline_cli(
assert ret == 0
def test_repro_pipelines(
tmp_dir,
dvc,
copy_script,
):
def test_repro_pipelines(tmp_dir, dvc, copy_script):
foo_stage, bar_stage = tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
file1_stage = dvc.run(
outs=["file1"],
@@ -1021,11 +960,7 @@ def test_repro_pipelines(
}
def test_repro_pipelines_cli(
tmp_dir,
dvc,
copy_script,
):
def test_repro_pipelines_cli(tmp_dir, dvc, copy_script):
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
dvc.run(
outs=["file1"],
@@ -1062,18 +997,11 @@ def test_freeze_non_existing(dvc, target):
assert ret != 0
def test_repro_frozen_callback(
tmp_dir,
dvc,
copy_script,
):
def test_repro_frozen_callback(tmp_dir, dvc, copy_script):
tmp_dir.gen("foo", "foo")
# NOTE: purposefully not specifying deps or outs
# to create a callback stage.
stage = dvc.run(
cmd="python copy.py foo file1",
name="copy-FOO-file1",
)
stage = dvc.run(cmd="python copy.py foo file1", name="copy-FOO-file1")
stages = dvc.reproduce(stage.addressing)
assert len(stages) == 1
@@ -1087,11 +1015,7 @@ def test_repro_frozen_callback(
assert len(stages) == 1
def test_repro_frozen_unchanged(
tmp_dir,
dvc,
copy_script,
):
def test_repro_frozen_unchanged(tmp_dir, dvc, copy_script):
"""
Check that freezing/unfreezing doesn't affect stage state
"""
@@ -1155,11 +1079,7 @@ def test_repro_metrics_add_unchanged(tmp_dir, dvc, copy_script):
assert len(stages) == 0
def test_repro_phony(
tmp_dir,
dvc,
copy_script,
):
def test_repro_phony(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
dvc.run(
@@ -1176,11 +1096,7 @@ def test_repro_phony(
assert filecmp.cmp("file1", "bar", shallow=False)
def test_non_existing_output(
tmp_dir,
dvc,
copy_script,
):
def test_non_existing_output(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
@@ -1195,11 +1111,7 @@ def test_non_existing_output(
dvc.reproduce(stage.addressing)
def test_repro_data_source(
tmp_dir,
dvc,
copy_script,
):
def test_repro_data_source(tmp_dir, dvc, copy_script):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run(
@@ -1216,11 +1128,7 @@ def test_repro_data_source(
assert stages[0].outs[0].hash_info.value == file_md5("bar")
def test_repro_changed_dir(
tmp_dir,
dvc,
copy_script,
):
def test_repro_changed_dir(tmp_dir, dvc, copy_script):
tmp_dir.gen({"foo": "foo", "bar": "bar"})
shutil.copyfile("foo", "file")
@@ -1241,11 +1149,7 @@ def test_repro_changed_dir(
assert len(stages) == 1
def test_repro_changed_dir_data(
tmp_dir,
dvc,
copy_script,
):
def test_repro_changed_dir_data(tmp_dir, dvc, copy_script):
tmp_dir.gen({"data": {"foo": "foo"}, "bar": "bar"})
stage = dvc.run(
outs=["dir"],
+2 -8
View File
@@ -754,10 +754,7 @@ def test_add_file_in_symlink_dir(make_tmp_dir, tmp_dir, dvc):
def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys):
tmp_dir.gen("foo", "foo")
mocker.patch(
"dvc_data.hashfile.checkout.test_links",
return_value=[],
)
mocker.patch("dvc_data.hashfile.checkout.test_links", return_value=[])
dvc.add("foo")
err = capsys.readouterr()[1]
assert "reconfigure cache types" in err
@@ -981,10 +978,7 @@ def test_add_does_not_remove_stage_file_on_failure(tmp_dir, dvc, mocker, target)
dvcfile_contents = (tmp_dir / stage.path).read_text()
exc_msg = f"raising error from mocked '{target}'"
mocker.patch(
target,
side_effect=DvcException(exc_msg),
)
mocker.patch(target, side_effect=DvcException(exc_msg))
with pytest.raises(DvcException, match=exc_msg):
dvc.add("foo")
+2 -12
View File
@@ -186,12 +186,7 @@ def test_checkout_empty_dir(tmp_dir, dvc):
def test_checkout_not_cached_file(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
dvc.run(
cmd="cp foo bar",
deps=["foo"],
outs_no_cache=["bar"],
name="copy-file",
)
dvc.run(cmd="cp foo bar", deps=["foo"], outs_no_cache=["bar"], name="copy-file")
stats = dvc.checkout(force=True)
assert not any(stats.values())
@@ -520,12 +515,7 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link):
def test_checkout_with_deps(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo"})
dvc.run(
cmd="echo foo > bar",
outs=["bar"],
deps=["foo"],
name="copy-file",
)
dvc.run(cmd="echo foo > bar", outs=["bar"], deps=["foo"], name="copy-file")
(tmp_dir / "bar").unlink()
(tmp_dir / "foo").unlink()
+2 -7
View File
@@ -10,9 +10,7 @@ from dvc.cli import main
from dvc.exceptions import CheckoutError
from dvc.repo.open_repo import clean_repos
from dvc.stage.exceptions import StageNotFound
from dvc.testing.remote_tests import ( # noqa: F401
TestRemote,
)
from dvc.testing.remote_tests import TestRemote # noqa: F401
from dvc.utils.fs import remove
from dvc_data.hashfile.db import HashFileDB
from dvc_data.hashfile.db.local import LocalHashFileDB
@@ -177,10 +175,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
assert bar in caplog.text
caplog.clear()
assert dvc.status(cloud=True) == {
"bar": "missing",
"foo": "missing",
}
assert dvc.status(cloud=True) == {"bar": "missing", "foo": "missing"}
assert header not in caplog.text
assert foo not in caplog.text
assert bar not in caplog.text
+6 -23
View File
@@ -84,10 +84,7 @@ def test_directory(M, tmp_dir, dvc, scm):
assert dvc.data_status(granular=True, untracked_files="all") == {
**EMPTY_STATUS,
"committed": {
"added": M.unordered(
join("dir", "bar"),
join("dir", "foobar"),
),
"added": M.unordered(join("dir", "bar"), join("dir", "foobar")),
"modified": [join("dir", "")],
},
"uncommitted": {
@@ -137,10 +134,7 @@ def test_new_empty_git_repo(M, tmp_dir, scm):
dvc = Repo.init()
assert dvc.data_status() == {
**EMPTY_STATUS,
"git": M.dict(
is_empty=True,
is_dirty=True,
),
"git": M.dict(is_empty=True, is_dirty=True),
}
@@ -180,10 +174,7 @@ def test_outs_with_no_hashes(M, tmp_dir, dvc, scm):
dvc.stage.add(single_stage=True, outs=["bar"])
dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
expected_output = {
**EMPTY_STATUS,
"git": M.dict(),
}
expected_output = {**EMPTY_STATUS, "git": M.dict()}
assert dvc.data_status() == expected_output
assert dvc.data_status(granular=True) == expected_output
@@ -251,10 +242,7 @@ def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
"untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
"committed": {"added": M.unordered("foobar", join("dir", ""))},
"uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))},
"not_in_cache": M.unordered(
"foobar",
join("dir", ""),
),
"not_in_cache": M.unordered("foobar", join("dir", "")),
"git": M.dict(),
}
@@ -297,10 +285,7 @@ def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
}
assert dvc.data_status(granular=True) == {
**EMPTY_STATUS,
"not_in_cache": M.unordered(
"foobar",
join("dir", ""),
),
"not_in_cache": M.unordered("foobar", join("dir", "")),
"uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))},
"git": M.dict(),
"unchanged": M.unordered("foobar", join("dir", "")),
@@ -389,9 +374,7 @@ def test_missing_dir_object_from_index(M, tmp_dir, dvc, scm):
}
assert dvc.data_status(granular=True) == {
**EMPTY_STATUS,
"committed": {
"modified": [join("dir", "")],
},
"committed": {"modified": [join("dir", "")]},
"uncommitted": {"unknown": [join("dir", "foobar")]},
"not_in_cache": [join("dir", "")],
"git": M.dict(),
+1 -4
View File
@@ -106,10 +106,7 @@ def test_load_all_multistage(tmp_dir, dvc):
metrics=["bar2"],
always_changed=True,
)
assert set(load_file(dvc, PROJECT_FILE).stages.values()) == {
stage2,
stage1,
}
assert set(load_file(dvc, PROJECT_FILE).stages.values()) == {stage2, stage1}
def test_load_all_singlestage(tmp_dir, dvc):
+3 -12
View File
@@ -93,10 +93,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
dest = tmp_dir / "file"
with external_repo(os.fspath(erepo_dir)) as repo:
repo.dvcfs.get(
"subdir/file",
os.fspath(dest),
)
repo.dvcfs.get("subdir/file", os.fspath(dest))
assert dest.is_file()
assert dest.read_text() == "contents"
@@ -191,10 +188,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.dvcfs.get(
"dir",
os.fspath(tmp_dir / "out"),
)
repo.dvcfs.get("dir", os.fspath(tmp_dir / "out"))
expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"}
assert (tmp_dir / "out").read_text() == expected_files
@@ -246,9 +240,6 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo:
repo.dvcfs.get(
"dir",
os.fspath(tmp_dir / "out"),
)
repo.dvcfs.get("dir", os.fspath(tmp_dir / "out"))
# subrepo files should not be here
assert (tmp_dir / "out").read_text() == scm_files

Some files were not shown because too many files have changed in this diff Show More