nit: fit statement/expr in the same line instead of hanging (#10267)

The code are wrapped due to us using line-length=79 before, but now some
of the code can easily fit in 88 chars.

The formatter did not change this automatically when we changed line-length,
because of magic trailing commas that gets added when they turn into multiple lines.

Black/ruff wrapping them in parens and splitting them into multiple lines
was making it difficult to read.
This commit is contained in:
skshetry
2024-01-30 18:23:04 +05:45
committed by GitHub
parent 422caa13c4
commit 9a1bcd7f1d
144 changed files with 342 additions and 1707 deletions
+1 -4
View File
@@ -5,10 +5,7 @@ from PyInstaller.utils.hooks import collect_submodules, is_module_or_submodule
# Celery dynamically imports most celery internals at runtime # Celery dynamically imports most celery internals at runtime
# pyinstaller hook must expose all modules loaded by # pyinstaller hook must expose all modules loaded by
# kombu.utils.imports:symbol_by_name() # kombu.utils.imports:symbol_by_name()
_EXCLUDES = ( _EXCLUDES = ("celery.bin", "celery.contrib")
"celery.bin",
"celery.contrib",
)
hiddenimports = collect_submodules( hiddenimports = collect_submodules(
"celery", "celery",
filter=lambda name: not any( filter=lambda name: not any(
+2 -10
View File
@@ -57,11 +57,7 @@ def get_url(path, repo=None, rev=None, remote=None):
class _OpenContextManager(GCM): class _OpenContextManager(GCM):
def __init__(self, func, args, kwds): def __init__(self, func, args, kwds):
self.gen = func(*args, **kwds) self.gen = func(*args, **kwds)
self.func, self.args, self.kwds = ( # type: ignore[assignment] self.func, self.args, self.kwds = (func, args, kwds) # type: ignore[assignment]
func,
args,
kwds,
)
def __getattr__(self, name): def __getattr__(self, name):
raise AttributeError("dvc.api.open() should be used in a with statement.") raise AttributeError("dvc.api.open() should be used in a with statement.")
@@ -262,11 +258,7 @@ def _open(
fs_path = fs.from_os_path(path) fs_path = fs.from_os_path(path)
try: try:
with fs.open( with fs.open(fs_path, mode=mode, encoding=encoding) as fobj:
fs_path,
mode=mode,
encoding=encoding,
) as fobj:
yield fobj yield fobj
except FileNotFoundError as exc: except FileNotFoundError as exc:
raise FileMissingError(path) from exc raise FileMissingError(path) from exc
+2 -8
View File
@@ -144,15 +144,9 @@ def migrate_2_to_3(repo: "Repo", dry: bool = False):
) )
return return
with TqdmCallback( with TqdmCallback(desc="Computing DVC 3.0 hashes", unit="files") as cb:
desc="Computing DVC 3.0 hashes",
unit="files",
) as cb:
migration = prepare(src, dest, callback=cb) migration = prepare(src, dest, callback=cb)
with TqdmCallback( with TqdmCallback(desc="Migrating to DVC 3.0 cache", unit="files") as cb:
desc="Migrating to DVC 3.0 cache",
unit="files",
) as cb:
count = migrate(migration, callback=cb) count = migrate(migration, callback=cb)
ui.write(f"Migrated {count} files to DVC 3.0 cache location.") ui.write(f"Migrated {count} files to DVC 3.0 cache location.")
+1 -5
View File
@@ -107,8 +107,4 @@ def _add_run_common(parser):
default=None, default=None,
help="Custom commit message to use when committing the experiment.", help="Custom commit message to use when committing the experiment.",
) )
parser.add_argument( parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
"-M", # obsolete
dest="message",
help=argparse.SUPPRESS,
)
+1 -5
View File
@@ -93,9 +93,5 @@ def add_parser(experiments_subparsers, parent_parser):
default=None, default=None,
help="Custom commit message to use when committing the experiment.", help="Custom commit message to use when committing the experiment.",
) )
save_parser.add_argument( save_parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
"-M", # obsolete
dest="message",
help=argparse.SUPPRESS,
)
save_parser.set_defaults(func=CmdExperimentsSave) save_parser.set_defaults(func=CmdExperimentsSave)
+1 -5
View File
@@ -119,11 +119,7 @@ def add_parser(subparsers, parent_parser):
"specified by '--remote') in the target repository." "specified by '--remote') in the target repository."
), ),
) )
list_parser.add_argument( list_parser.add_argument("--size", action="store_true", help="Show sizes.")
"--size",
action="store_true",
help="Show sizes.",
)
list_parser.add_argument( list_parser.add_argument(
"path", "path",
nargs="?", nargs="?",
+2 -9
View File
@@ -38,16 +38,9 @@ def add_parser(subparsers, parent_parser):
"url", help="See `dvc import-url -h` for full list of supported URLs." "url", help="See `dvc import-url -h` for full list of supported URLs."
) )
lsurl_parser.add_argument( lsurl_parser.add_argument(
"-R", "-R", "--recursive", action="store_true", help="Recursively list files."
"--recursive",
action="store_true",
help="Recursively list files.",
)
lsurl_parser.add_argument(
"--size",
action="store_true",
help="Show sizes.",
) )
lsurl_parser.add_argument("--size", action="store_true", help="Show sizes.")
lsurl_parser.add_argument( lsurl_parser.add_argument(
"--fs-config", "--fs-config",
type=str, type=str,
+3 -15
View File
@@ -34,21 +34,12 @@ def _show_json(
name = renderer.name name = renderer.name
data[name] = to_json(renderer, split) data[name] = to_json(renderer, split)
all_errors.extend( all_errors.extend(
{ {"name": name, "rev": rev, "source": source, **encode_exception(e)}
"name": name,
"rev": rev,
"source": source,
**encode_exception(e),
}
for rev, per_rev_src_errors in src_errors.items() for rev, per_rev_src_errors in src_errors.items()
for source, e in per_rev_src_errors.items() for source, e in per_rev_src_errors.items()
) )
all_errors.extend( all_errors.extend(
{ {"name": name, "rev": rev, **encode_exception(e)}
"name": name,
"rev": rev,
**encode_exception(e),
}
for rev, e in def_errors.items() for rev, e in def_errors.items()
) )
@@ -101,10 +92,7 @@ class CmdPlots(CmdBase):
return 1 return 1
try: try:
plots_data = self._func( plots_data = self._func(targets=self.args.targets, props=self._props())
targets=self.args.targets,
props=self._props(),
)
if not plots_data and not self.args.json: if not plots_data and not self.args.json:
ui.error_write( ui.error_write(
+1 -8
View File
@@ -2,14 +2,7 @@ from dvc.cli import formatter
from dvc.cli.utils import append_doc_link from dvc.cli.utils import append_doc_link
from dvc.commands.queue import kill, logs, remove, start, status, stop from dvc.commands.queue import kill, logs, remove, start, status, stop
SUB_COMMANDS = [ SUB_COMMANDS = [start, stop, status, logs, remove, kill]
start,
stop,
status,
logs,
remove,
kill,
]
def add_parser(subparsers, parent_parser): def add_parser(subparsers, parent_parser):
+1 -5
View File
@@ -45,9 +45,5 @@ def add_parser(queue_subparsers, parent_parser):
), ),
action="store_true", action="store_true",
) )
queue_logs_parser.add_argument( queue_logs_parser.add_argument("task", help="Task to show.", metavar="<task>")
"task",
help="Task to show.",
metavar="<task>",
)
queue_logs_parser.set_defaults(func=CmdQueueLogs) queue_logs_parser.set_defaults(func=CmdQueueLogs)
+1 -7
View File
@@ -9,13 +9,7 @@ from collections.abc import (
) )
from itertools import chain, repeat, zip_longest from itertools import chain, repeat, zip_longest
from operator import itemgetter from operator import itemgetter
from typing import ( from typing import TYPE_CHECKING, Any, Optional, Union, overload
TYPE_CHECKING,
Any,
Optional,
Union,
overload,
)
from funcy import reraise from funcy import reraise
+1 -6
View File
@@ -107,12 +107,7 @@ def _posix_detached_subprocess(args: Sequence[str], **kwargs) -> int:
os.close(read_end) os.close(read_end)
return int(pid_str) return int(pid_str)
proc = subprocess.Popen( proc = subprocess.Popen(args, shell=False, close_fds=True, **kwargs) # noqa: S603
args,
shell=False, # noqa: S603
close_fds=True,
**kwargs,
)
os.close(read_end) os.close(read_end)
os.write(write_end, str(proc.pid).encode("utf8")) os.write(write_end, str(proc.pid).encode("utf8"))
os.close(write_end) os.close(write_end)
+1 -9
View File
@@ -59,15 +59,7 @@ def loads_from(stage, s_list, erepo=None, fs_config=None, db=None):
info = {RepoDependency.PARAM_REPO: erepo} if erepo else {} info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
if db: if db:
info.update({"db": db}) info.update({"db": db})
return [ return [_get(stage, s, info.copy(), fs_config=fs_config) for s in s_list]
_get(
stage,
s,
info.copy(),
fs_config=fs_config,
)
for s in s_list
]
def _merge_params(s_list) -> dict[str, list[str]]: def _merge_params(s_list) -> dict[str, list[str]]:
+2 -8
View File
@@ -76,10 +76,7 @@ class ParamsDependency(Dependency):
self.params = list(params) if params else [] self.params = list(params) if params else []
hash_info = HashInfo() hash_info = HashInfo()
if isinstance(params, dict): if isinstance(params, dict):
hash_info = HashInfo( hash_info = HashInfo(self.PARAM_PARAMS, params) # type: ignore[arg-type]
self.PARAM_PARAMS,
params, # type: ignore[arg-type]
)
repo = repo or stage.repo repo = repo or stage.repo
path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE) path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE)
super().__init__(stage, path, repo=repo) super().__init__(stage, path, repo=repo)
@@ -103,10 +100,7 @@ class ParamsDependency(Dependency):
for param in self.params: for param in self.params:
if param in values: if param in values:
info[param] = values[param] info[param] = values[param]
self.hash_info = HashInfo( self.hash_info = HashInfo(self.PARAM_PARAMS, info) # type: ignore[arg-type]
self.PARAM_PARAMS,
info, # type: ignore[arg-type]
)
def read_params( def read_params(
self, flatten: bool = True, **kwargs: typing.Any self, flatten: bool = True, **kwargs: typing.Any
+1 -9
View File
@@ -1,14 +1,6 @@
import contextlib import contextlib
import os import os
from typing import ( from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, TypeVar, Union
TYPE_CHECKING,
Any,
Callable,
ClassVar,
Optional,
TypeVar,
Union,
)
from dvc.exceptions import DvcException from dvc.exceptions import DvcException
from dvc.log import logger from dvc.log import logger
+2 -12
View File
@@ -52,10 +52,7 @@ def download(
from .callbacks import TqdmCallback from .callbacks import TqdmCallback
with TqdmCallback( with TqdmCallback(desc=f"Downloading {fs.name(fs_path)}", unit="files") as cb:
desc=f"Downloading {fs.name(fs_path)}",
unit="files",
) as cb:
# NOTE: We use dvc-objects generic.copy over fs.get since it makes file # NOTE: We use dvc-objects generic.copy over fs.get since it makes file
# download atomic and avoids fsspec glob/regex path expansion. # download atomic and avoids fsspec glob/regex path expansion.
if fs.isdir(fs_path): if fs.isdir(fs_path):
@@ -76,14 +73,7 @@ def download(
lfs_prefetch(fs, from_infos) lfs_prefetch(fs, from_infos)
cb.set_size(len(from_infos)) cb.set_size(len(from_infos))
jobs = jobs or fs.jobs jobs = jobs or fs.jobs
generic.copy( generic.copy(fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs)
fs,
from_infos,
localfs,
to_infos,
callback=cb,
batch_size=jobs,
)
return len(to_infos) return len(to_infos)
+1 -3
View File
@@ -22,9 +22,7 @@ class DataFileSystem(FileSystem):
return config return config
@functools.cached_property @functools.cached_property
def fs( def fs(self) -> "_DataFileSystem":
self,
) -> "_DataFileSystem":
from dvc_data.fs import DataFileSystem as _DataFileSystem from dvc_data.fs import DataFileSystem as _DataFileSystem
return _DataFileSystem(**self.fs_args) return _DataFileSystem(**self.fs_args)
+1 -3
View File
@@ -39,9 +39,7 @@ class GitFileSystem(FileSystem):
) )
@functools.cached_property @functools.cached_property
def fs( def fs(self) -> "FsspecGitFileSystem":
self,
) -> "FsspecGitFileSystem":
from scmrepo.fs import GitFileSystem as FsspecGitFileSystem from scmrepo.fs import GitFileSystem as FsspecGitFileSystem
return FsspecGitFileSystem(**self.fs_args) return FsspecGitFileSystem(**self.fs_args)
+1 -3
View File
@@ -185,9 +185,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
if self._tmp_dir is not None: if self._tmp_dir is not None:
# Under Windows file path length is limited so we hash it # Under Windows file path length is limited so we hash it
hasher = hashlib.md5( # noqa: S324 hasher = hashlib.md5(self._claimfile.encode()) # noqa: S324
self._claimfile.encode()
)
filename = hasher.hexdigest() filename = hasher.hexdigest()
self._claimfile = os.path.join(self._tmp_dir, filename + ".lock") self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
+9 -43
View File
@@ -167,11 +167,7 @@ def _merge_data(s_list):
@collecting @collecting
def load_from_pipeline(stage, data, typ="outs"): def load_from_pipeline(stage, data, typ="outs"):
if typ not in ( if typ not in (stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS):
stage.PARAM_OUTS,
stage.PARAM_METRICS,
stage.PARAM_PLOTS,
):
raise ValueError(f"'{typ}' key is not allowed for pipeline files.") raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
metric = typ == stage.PARAM_METRICS metric = typ == stage.PARAM_METRICS
@@ -197,24 +193,13 @@ def load_from_pipeline(stage, data, typ="outs"):
], ],
) )
yield _get( yield _get(stage, path, info={}, plot=plt_d or plot, metric=metric, **extra)
stage,
path,
info={},
plot=plt_d or plot,
metric=metric,
**extra,
)
def split_file_meta_from_cloud(entry: dict) -> dict: def split_file_meta_from_cloud(entry: dict) -> dict:
if remote_name := entry.pop(Meta.PARAM_REMOTE, None): if remote_name := entry.pop(Meta.PARAM_REMOTE, None):
remote_meta = {} remote_meta = {}
for key in ( for key in (S3_PARAM_CHECKSUM, HDFS_PARAM_CHECKSUM, Meta.PARAM_VERSION_ID):
S3_PARAM_CHECKSUM,
HDFS_PARAM_CHECKSUM,
Meta.PARAM_VERSION_ID,
):
if value := entry.pop(key, None): if value := entry.pop(key, None):
remote_meta[key] = value remote_meta[key] = value
@@ -433,10 +418,7 @@ class Output:
hash_name = meta_name = self.fs.PARAM_CHECKSUM hash_name = meta_name = self.fs.PARAM_CHECKSUM
assert hash_name assert hash_name
hash_info = HashInfo( hash_info = HashInfo(name=hash_name, value=getattr(self.meta, meta_name, None))
name=hash_name,
value=getattr(self.meta, meta_name, None),
)
return hash_name, hash_info return hash_name, hash_info
def _compute_meta_hash_info_from_files(self) -> None: def _compute_meta_hash_info_from_files(self) -> None:
@@ -515,10 +497,7 @@ class Output:
if self.fs.isabs(self.def_path): if self.fs.isabs(self.def_path):
return False return False
return self.repo and self.fs.isin( return self.repo and self.fs.isin(self.fs_path, self.repo.root_dir)
self.fs_path,
self.repo.root_dir,
)
@property @property
def use_scm_ignore(self): def use_scm_ignore(self):
@@ -813,20 +792,13 @@ class Output:
def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]: def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path)) prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
staging, _, obj = self._build( staging, _, obj = self._build(
self.cache, self.cache, self.fs_path, self.fs, self.hash_name, ignore=self.dvcignore
self.fs_path,
self.fs,
self.hash_name,
ignore=self.dvcignore,
) )
assert isinstance(obj, Tree) assert isinstance(obj, Tree)
save_obj = obj.filter(prefix) save_obj = obj.filter(prefix)
assert isinstance(save_obj, Tree) assert isinstance(save_obj, Tree)
checkout_obj = save_obj.get_obj(self.cache, prefix) checkout_obj = save_obj.get_obj(self.cache, prefix)
with TqdmCallback( with TqdmCallback(desc=f"Committing {self} to cache", unit="file") as cb:
desc=f"Committing {self} to cache",
unit="file",
) as cb:
otransfer( otransfer(
staging, staging,
self.cache, self.cache,
@@ -1286,10 +1258,7 @@ class Output:
self.hash_info = merged.hash_info self.hash_info = merged.hash_info
self.files = None self.files = None
self.meta = Meta( self.meta = Meta(size=du(self.cache, merged), nfiles=len(merged))
size=du(self.cache, merged),
nfiles=len(merged),
)
def unstage(self, path: str) -> tuple["Meta", "Tree"]: def unstage(self, path: str) -> tuple["Meta", "Tree"]:
from pygtrie import Trie from pygtrie import Trie
@@ -1429,10 +1398,7 @@ class Output:
assert staging assert staging
assert obj.hash_info assert obj.hash_info
with TqdmCallback( with TqdmCallback(desc=f"Adding {self} to cache", unit="file") as cb:
desc=f"Adding {self} to cache",
unit="file",
) as cb:
otransfer( otransfer(
staging, staging,
self.cache, self.cache,
+1 -7
View File
@@ -3,13 +3,7 @@ import os
from collections.abc import Mapping, Sequence from collections.abc import Mapping, Sequence
from copy import deepcopy from copy import deepcopy
from itertools import product from itertools import product
from typing import ( from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
TYPE_CHECKING,
Any,
NamedTuple,
Optional,
Union,
)
from funcy import collecting, first, isa, join, reraise from funcy import collecting, first, isa, join, reraise
+2 -8
View File
@@ -28,9 +28,7 @@ class ImageConverter(Converter):
return img_path return img_path
@staticmethod @staticmethod
def _encode_image( def _encode_image(image_data: bytes) -> str:
image_data: bytes,
) -> str:
base64_str = base64.b64encode(image_data).decode() base64_str = base64.b64encode(image_data).decode()
return f"data:image;base64,{base64_str}" return f"data:image;base64,{base64_str}"
@@ -57,10 +55,6 @@ class ImageConverter(Converter):
) )
else: else:
src = self._encode_image(image_data) src = self._encode_image(image_data)
datapoint = { datapoint = {REVISION: revision, FILENAME: filename, SRC: src}
REVISION: revision,
FILENAME: filename,
SRC: src,
}
datapoints.append(datapoint) datapoints.append(datapoint)
return datapoints, properties return datapoints, properties
+4 -16
View File
@@ -41,11 +41,7 @@ def _file_field(*args):
yield file, field yield file, field
def _find( def _find(filename: str, field: str, data_series: list[tuple[str, str, Any]]):
filename: str,
field: str,
data_series: list[tuple[str, str, Any]],
):
for data_file, data_field, data in data_series: for data_file, data_field, data in data_series:
if data_file == filename and data_field == field: if data_file == filename and data_field == field:
return data_file, data_field, data return data_file, data_field, data
@@ -202,10 +198,7 @@ class VegaConverter(Converter):
# assign "step" if no x provided # assign "step" if no x provided
if not xs: if not xs:
x_file, x_field = ( x_file, x_field = None, INDEX
None,
INDEX,
)
else: else:
x_file, x_field = xs[0] x_file, x_field = xs[0]
@@ -244,10 +237,7 @@ class VegaConverter(Converter):
common_prefix_len = 0 common_prefix_len = 0
props_update["anchors_y_definitions"] = [ props_update["anchors_y_definitions"] = [
{ {FILENAME: _get_short_y_file(y_file, common_prefix_len), FIELD: y_field}
FILENAME: _get_short_y_file(y_file, common_prefix_len),
FIELD: y_field,
}
for y_file, y_field in ys for y_file, y_field in ys
] ]
@@ -299,9 +289,7 @@ class VegaConverter(Converter):
return all_datapoints, properties return all_datapoints, properties
def convert( def convert(self):
self,
):
""" """
Convert the data. Fill necessary fields ('x', 'y') and return both Convert the data. Fill necessary fields ('x', 'y') and return both
generated datapoints and updated properties. `x`, `y` values and labels generated datapoints and updated properties. `x`, `y` values and labels
+5 -28
View File
@@ -3,12 +3,7 @@ from collections import defaultdict
from collections.abc import Iterable from collections.abc import Iterable
from contextlib import AbstractContextManager, contextmanager from contextlib import AbstractContextManager, contextmanager
from functools import wraps from functools import wraps
from typing import ( from typing import TYPE_CHECKING, Callable, Optional, Union
TYPE_CHECKING,
Callable,
Optional,
Union,
)
from dvc.exceptions import ( from dvc.exceptions import (
DvcException, DvcException,
@@ -179,14 +174,8 @@ class Repo:
self.root_dir: str self.root_dir: str
self.dvc_dir: Optional[str] self.dvc_dir: Optional[str]
( (self.root_dir, self.dvc_dir) = self._get_repo_dirs(
self.root_dir, root_dir=root_dir, fs=self.fs, uninitialized=uninitialized, scm=scm
self.dvc_dir,
) = self._get_repo_dirs(
root_dir=root_dir,
fs=self.fs,
uninitialized=uninitialized,
scm=scm,
) )
self._uninitialized = uninitialized self._uninitialized = uninitialized
@@ -271,11 +260,7 @@ class Repo:
# subrepo # subrepo
relparts = self.fs.relparts(self.root_dir, "/") relparts = self.fs.relparts(self.root_dir, "/")
dvc_dir = os.path.join( dvc_dir = os.path.join(self.scm.root_dir, *relparts, self.DVC_DIR)
self.scm.root_dir,
*relparts,
self.DVC_DIR,
)
if os.path.exists(dvc_dir): if os.path.exists(dvc_dir):
return dvc_dir return dvc_dir
@@ -651,15 +636,7 @@ class Repo:
btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None) btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None)
md5 = hashlib.md5( # noqa: S324 md5 = hashlib.md5( # noqa: S324
str( str((root_dir, btime, getpass.getuser(), version_tuple[0], salt)).encode()
(
root_dir,
btime,
getpass.getuser(),
version_tuple[0],
salt,
)
).encode()
) )
repo_token = md5.hexdigest() repo_token = md5.hexdigest()
return os.path.join(repos_dir, repo_token) return os.path.join(repos_dir, repo_token)
+1 -6
View File
@@ -1,12 +1,7 @@
import os import os
from collections.abc import Iterator from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from typing import ( from typing import TYPE_CHECKING, NamedTuple, Optional, Union
TYPE_CHECKING,
NamedTuple,
Optional,
Union,
)
from dvc.exceptions import ( from dvc.exceptions import (
CacheLinkError, CacheLinkError,
+2 -10
View File
@@ -98,10 +98,7 @@ class Artifacts:
def read(self) -> dict[str, dict[str, Artifact]]: def read(self) -> dict[str, dict[str, Artifact]]:
"""Read artifacts from dvc.yaml.""" """Read artifacts from dvc.yaml."""
artifacts: dict[str, dict[str, Artifact]] = {} artifacts: dict[str, dict[str, Artifact]] = {}
for ( for dvcfile, dvcfile_artifacts in self.repo.index._artifacts.items():
dvcfile,
dvcfile_artifacts,
) in self.repo.index._artifacts.items():
dvcyaml = relpath(dvcfile, self.repo.root_dir) dvcyaml = relpath(dvcfile, self.repo.root_dir)
artifacts[dvcyaml] = {} artifacts[dvcyaml] = {}
for name, value in dvcfile_artifacts.items(): for name, value in dvcfile_artifacts.items():
@@ -188,12 +185,7 @@ class Artifacts:
out = resolve_output(path, out, force=force) out = resolve_output(path, out, force=force)
fs = self.repo.dvcfs fs = self.repo.dvcfs
fs_path = fs.from_os_path(path) fs_path = fs.from_os_path(path)
count = fs_download( count = fs_download(fs, fs_path, os.path.abspath(out), jobs=jobs)
fs,
fs_path,
os.path.abspath(out),
jobs=jobs,
)
return count, out return count, out
@staticmethod @staticmethod
+1 -5
View File
@@ -5,11 +5,7 @@ def check_missing(repo, rev=None, max_size=None, types=None):
from dvc_data.index import StorageKeyError from dvc_data.index import StorageKeyError
with repo.switch(rev or "workspace"): with repo.switch(rev or "workspace"):
idx = repo.index.targets_view( idx = repo.index.targets_view(None, max_size=max_size, types=types)
None,
max_size=max_size,
types=types,
)
index = idx.data["repo"] index = idx.data["repo"]
+6 -29
View File
@@ -122,42 +122,23 @@ def checkout( # noqa: C901
def onerror(target, exc): def onerror(target, exc):
if target and isinstance( if target and isinstance(
exc, exc,
( (StageFileDoesNotExistError, StageFileBadNameError, NoOutputOrStageError),
StageFileDoesNotExistError,
StageFileBadNameError,
NoOutputOrStageError,
),
): ):
raise CheckoutErrorSuggestGit(target) from exc raise CheckoutErrorSuggestGit(target) from exc
raise raise
view = self.index.targets_view( view = self.index.targets_view(
targets, targets, recursive=recursive, with_deps=with_deps, onerror=onerror
recursive=recursive,
with_deps=with_deps,
onerror=onerror,
) )
with ui.progress( with ui.progress(unit="entry", desc="Building workspace index", leave=True) as pb:
unit="entry",
desc="Building workspace index",
leave=True,
) as pb:
old = build_data_index( old = build_data_index(
view, view, self.root_dir, self.fs, compute_hash=True, callback=pb.as_callback()
self.root_dir,
self.fs,
compute_hash=True,
callback=pb.as_callback(),
) )
new = view.data["repo"] new = view.data["repo"]
with ui.progress( with ui.progress(desc="Comparing indexes", unit="entry", leave=True) as pb:
desc="Comparing indexes",
unit="entry",
leave=True,
) as pb:
diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback()) diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback())
if not force: if not force:
@@ -175,11 +156,7 @@ def checkout( # noqa: C901
if self.fs.isin_or_eq(dest_path, out_path): if self.fs.isin_or_eq(dest_path, out_path):
failed.add(out_path) failed.add(out_path)
with ui.progress( with ui.progress(unit="file", desc="Applying changes", leave=True) as pb:
unit="file",
desc="Applying changes",
leave=True,
) as pb:
apply( apply(
diff, diff,
self.root_dir, self.root_dir,
+4 -17
View File
@@ -154,10 +154,7 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]: def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
from .index import build_data_index from .index import build_data_index
with ui.progress( with ui.progress(desc="Building workspace index", unit="entry") as pb:
desc="Building workspace index",
unit="entry",
) as pb:
workspace = build_data_index( workspace = build_data_index(
repo.index, repo.index,
repo.root_dir, repo.root_dir,
@@ -187,10 +184,7 @@ def _diff_head_to_index(
with repo.switch(head): with repo.switch(head):
head_index = repo.index.data["repo"] head_index = repo.index.data["repo"]
with ui.progress( with ui.progress(desc="Calculating diff between head/index", unit="entry") as pb:
desc="Calculating diff between head/index",
unit="entry",
) as pb:
return _diff(head_index, index, callback=pb.as_callback(), **kwargs) return _diff(head_index, index, callback=pb.as_callback(), **kwargs)
@@ -223,18 +217,11 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
return [repo.fs.relpath(file, start) for file in files] return [repo.fs.relpath(file, start) for file in files]
def status( def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
repo: "Repo",
untracked_files: str = "no",
**kwargs: Any,
) -> Status:
from dvc.scm import NoSCMError, SCMError from dvc.scm import NoSCMError, SCMError
head = kwargs.pop("head", "HEAD") head = kwargs.pop("head", "HEAD")
uncommitted_diff = _diff_index_to_wtree( uncommitted_diff = _diff_index_to_wtree(repo, **kwargs)
repo,
**kwargs,
)
unchanged = set(uncommitted_diff.pop("unchanged", [])) unchanged = set(uncommitted_diff.pop("unchanged", []))
try: try:
+7 -33
View File
@@ -52,36 +52,22 @@ def _diff(old, new, data_keys, with_missing=False):
continue continue
if change.typ == ADD: if change.typ == ADD:
ret["added"].append( ret["added"].append({"path": _path(change.new), "hash": _hash(change.new)})
{
"path": _path(change.new),
"hash": _hash(change.new),
}
)
elif change.typ == DELETE: elif change.typ == DELETE:
ret["deleted"].append( ret["deleted"].append(
{ {"path": _path(change.old), "hash": _hash(change.old)}
"path": _path(change.old),
"hash": _hash(change.old),
}
) )
elif change.typ == MODIFY: elif change.typ == MODIFY:
ret["modified"].append( ret["modified"].append(
{ {
"path": _path(change.old), "path": _path(change.old),
"hash": { "hash": {"old": _hash(change.old), "new": _hash(change.new)},
"old": _hash(change.old),
"new": _hash(change.new),
},
} }
) )
elif change.typ == RENAME: elif change.typ == RENAME:
ret["renamed"].append( ret["renamed"].append(
{ {
"path": { "path": {"old": _path(change.old), "new": _path(change.new)},
"old": _path(change.old),
"new": _path(change.new),
},
"hash": _hash(change.old), "hash": _hash(change.old),
} }
) )
@@ -93,10 +79,7 @@ def _diff(old, new, data_keys, with_missing=False):
and not old.storage_map.cache_exists(change.old) and not old.storage_map.cache_exists(change.old)
): ):
ret["not in cache"].append( ret["not in cache"].append(
{ {"path": _path(change.old), "hash": _hash(change.old)}
"path": _path(change.old),
"hash": _hash(change.old),
}
) )
return ret if any(ret.values()) else {} return ret if any(ret.values()) else {}
@@ -137,11 +120,7 @@ def diff(
def onerror(target, _exc): def onerror(target, _exc):
missing_targets[rev].add(target) # noqa: B023 missing_targets[rev].add(target) # noqa: B023
view = self.index.targets_view( view = self.index.targets_view(targets, onerror=onerror, recursive=recursive)
targets,
onerror=onerror,
recursive=recursive,
)
data_keys.update(view.data_keys.get("repo", set())) data_keys.update(view.data_keys.get("repo", set()))
@@ -149,12 +128,7 @@ def diff(
from .index import build_data_index from .index import build_data_index
with ui.status("Building workspace index"): with ui.status("Building workspace index"):
data = build_data_index( data = build_data_index(view, self.root_dir, self.fs, compute_hash=True)
view,
self.root_dir,
self.fs,
compute_hash=True,
)
else: else:
data = view.data["repo"] data = view.data["repo"]
+5 -23
View File
@@ -131,16 +131,9 @@ class Experiments:
self._log_reproduced(results, tmp_dir=tmp_dir) self._log_reproduced(results, tmp_dir=tmp_dir)
return results return results
def queue_one( def queue_one(self, queue: "BaseStashQueue", **kwargs) -> "QueueEntry":
self,
queue: "BaseStashQueue",
**kwargs,
) -> "QueueEntry":
"""Queue a single experiment.""" """Queue a single experiment."""
return self.new( return self.new(queue, **kwargs)
queue,
**kwargs,
)
def reproduce_celery( def reproduce_celery(
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
@@ -148,16 +141,10 @@ class Experiments:
results: dict[str, str] = {} results: dict[str, str] = {}
if entries is None: if entries is None:
entries = list( entries = list(
chain( chain(self.celery_queue.iter_active(), self.celery_queue.iter_queued())
self.celery_queue.iter_active(),
self.celery_queue.iter_queued(),
)
) )
logger.debug( logger.debug("reproduce all these entries '%s'", entries)
"reproduce all these entries '%s'",
entries,
)
if not entries: if not entries:
return results return results
@@ -212,12 +199,7 @@ class Experiments:
else: else:
ui.write("Experiment results have been applied to your workspace.") ui.write("Experiment results have been applied to your workspace.")
def new( def new(self, queue: "BaseStashQueue", *args, **kwargs) -> "QueueEntry":
self,
queue: "BaseStashQueue",
*args,
**kwargs,
) -> "QueueEntry":
"""Create and enqueue a new experiment. """Create and enqueue a new experiment.
Experiment will be derived from the current workspace. Experiment will be derived from the current workspace.
+3 -4
View File
@@ -33,10 +33,9 @@ def apply(repo: "Repo", rev: str, **kwargs):
try: try:
exp_rev = resolve_rev(repo.scm, rev) exp_rev = resolve_rev(repo.scm, rev)
except RevError as exc: except RevError as exc:
( (exp_ref_info, queue_entry) = exps.celery_queue.get_ref_and_entry_by_names(rev)[
exp_ref_info, rev
queue_entry, ]
) = exps.celery_queue.get_ref_and_entry_by_names(rev)[rev]
if exp_ref_info: if exp_ref_info:
exp_rev = repo.scm.get_ref(str(exp_ref_info)) exp_rev = repo.scm.get_ref(str(exp_ref_info))
elif queue_entry: elif queue_entry:
+1 -4
View File
@@ -10,10 +10,7 @@ if TYPE_CHECKING:
@contextmanager @contextmanager
def switch_repo( def switch_repo(repo: "Repo", rev: str) -> Iterator[tuple["Repo", str]]:
repo: "Repo",
rev: str,
) -> Iterator[tuple["Repo", str]]:
"""Return a repo instance (brancher) switched to rev. """Return a repo instance (brancher) switched to rev.
If rev is the name of a running experiment, the returned instance will be If rev is the name of a running experiment, the returned instance will be
+2 -12
View File
@@ -3,11 +3,7 @@ import os
from collections.abc import Collection, Iterable, Iterator from collections.abc import Collection, Iterable, Iterator
from dataclasses import fields from dataclasses import fields
from datetime import datetime from datetime import datetime
from typing import ( from typing import TYPE_CHECKING, Optional, Union
TYPE_CHECKING,
Optional,
Union,
)
from funcy import first from funcy import first
from scmrepo.exceptions import SCMError as InnerSCMError from scmrepo.exceptions import SCMError as InnerSCMError
@@ -58,13 +54,7 @@ def collect_rev(
else: else:
orig_cwd = None orig_cwd = None
try: try:
data = _collect_rev( data = _collect_rev(repo, rev, param_deps=param_deps, force=force, **kwargs)
repo,
rev,
param_deps=param_deps,
force=force,
**kwargs,
)
if not (rev == "workspace" or param_deps or data.contains_error): if not (rev == "workspace" or param_deps or data.contains_error):
cache.put(data, force=True) cache.put(data, force=True)
return ExpState(rev=rev, data=data) return ExpState(rev=rev, data=data)
+1 -5
View File
@@ -51,11 +51,7 @@ class MultipleBranchError(DvcException):
class AmbiguousExpRefInfo(InvalidArgumentError): class AmbiguousExpRefInfo(InvalidArgumentError):
def __init__( def __init__(self, exp_name: str, exp_ref_list: Iterable["ExpRefInfo"]):
self,
exp_name: str,
exp_ref_list: Iterable["ExpRefInfo"],
):
msg = [ msg = [
( (
f"Ambiguous name '{exp_name}' refers to multiple experiments." f"Ambiguous name '{exp_name}' refers to multiple experiments."
+1 -8
View File
@@ -8,14 +8,7 @@ from contextlib import contextmanager
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from enum import IntEnum from enum import IntEnum
from itertools import chain from itertools import chain
from typing import ( from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
TYPE_CHECKING,
Any,
Callable,
NamedTuple,
Optional,
Union,
)
from scmrepo.exceptions import SCMError from scmrepo.exceptions import SCMError
+2 -11
View File
@@ -132,11 +132,7 @@ class TempDirExecutor(BaseLocalExecutor):
self._update_config(repo.config.read("local")) self._update_config(repo.config.read("local"))
def _update_config(self, update): def _update_config(self, update):
local_config = os.path.join( local_config = os.path.join(self.root_dir, self.dvc_dir, "config.local")
self.root_dir,
self.dvc_dir,
"config.local",
)
logger.debug("Writing experiments local config '%s'", local_config) logger.debug("Writing experiments local config '%s'", local_config)
if os.path.exists(local_config): if os.path.exists(local_config):
conf_obj = ConfigObj(local_config) conf_obj = ConfigObj(local_config)
@@ -188,12 +184,7 @@ class WorkspaceExecutor(BaseLocalExecutor):
self._detach_stack = ExitStack() self._detach_stack = ExitStack()
@classmethod @classmethod
def from_stash_entry( def from_stash_entry(cls, repo: "Repo", entry: "ExpStashEntry", **kwargs):
cls,
repo: "Repo",
entry: "ExpStashEntry",
**kwargs,
):
root_dir = repo.scm.root_dir root_dir = repo.scm.root_dir
executor: "WorkspaceExecutor" = cls._from_stash_entry( executor: "WorkspaceExecutor" = cls._from_stash_entry(
repo, entry, root_dir, **kwargs repo, entry, root_dir, **kwargs
+1 -6
View File
@@ -1,10 +1,5 @@
from collections.abc import Iterable, Mapping from collections.abc import Iterable, Mapping
from typing import ( from typing import TYPE_CHECKING, Any, Optional, Union
TYPE_CHECKING,
Any,
Optional,
Union,
)
from funcy import compact, group_by from funcy import compact, group_by
from scmrepo.git.backend.base import SyncStatus from scmrepo.git.backend.base import SyncStatus
+4 -20
View File
@@ -2,13 +2,7 @@ import os
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Collection, Generator, Iterable, Mapping from collections.abc import Collection, Generator, Iterable, Mapping
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from typing import ( from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
TYPE_CHECKING,
Any,
NamedTuple,
Optional,
Union,
)
from dvc_studio_client.post_live_metrics import get_studio_config from dvc_studio_client.post_live_metrics import get_studio_config
from funcy import retry from funcy import retry
@@ -275,12 +269,7 @@ class BaseStashQueue(ABC):
""" """
@abstractmethod @abstractmethod
def logs( def logs(self, rev: str, encoding: Optional[str] = None, follow: bool = False):
self,
rev: str,
encoding: Optional[str] = None,
follow: bool = False,
):
"""Print redirected output logs for an exp process. """Print redirected output logs for an exp process.
Args: Args:
@@ -336,9 +325,7 @@ class BaseStashQueue(ABC):
self._stash_commit_deps(*args, **kwargs) self._stash_commit_deps(*args, **kwargs)
# save additional repro command line arguments # save additional repro command line arguments
run_env = { run_env = {DVC_EXP_BASELINE_REV: baseline_rev}
DVC_EXP_BASELINE_REV: baseline_rev,
}
if not name: if not name:
name = get_random_exp_name(self.scm, baseline_rev) name = get_random_exp_name(self.scm, baseline_rev)
run_env[DVC_EXP_NAME] = name run_env[DVC_EXP_NAME] = name
@@ -502,10 +489,7 @@ class BaseStashQueue(ABC):
@staticmethod @staticmethod
@retry(180, errors=LockError, timeout=1) @retry(180, errors=LockError, timeout=1)
def get_stash_entry( def get_stash_entry(exp: "Experiments", queue_entry: QueueEntry) -> "ExpStashEntry":
exp: "Experiments",
queue_entry: QueueEntry,
) -> "ExpStashEntry":
stash = ExpStash(exp.scm, queue_entry.stash_ref) stash = ExpStash(exp.scm, queue_entry.stash_ref)
stash_rev = queue_entry.stash_rev stash_rev = queue_entry.stash_rev
with get_exp_rwlock(exp.repo, writes=[queue_entry.stash_ref]): with get_exp_rwlock(exp.repo, writes=[queue_entry.stash_ref]):
+7 -30
View File
@@ -4,12 +4,7 @@ import logging
import os import os
from collections import defaultdict from collections import defaultdict
from collections.abc import Collection, Generator, Mapping from collections.abc import Collection, Generator, Mapping
from typing import ( from typing import TYPE_CHECKING, NamedTuple, Optional, Union
TYPE_CHECKING,
NamedTuple,
Optional,
Union,
)
from celery.result import AsyncResult from celery.result import AsyncResult
from funcy import first from funcy import first
@@ -83,10 +78,7 @@ class LocalCeleryQueue(BaseStashQueue):
"dvc-exp-local", "dvc-exp-local",
wdir=self.wdir, wdir=self.wdir,
mkdir=True, mkdir=True,
include=[ include=["dvc.repo.experiments.queue.tasks", "dvc_task.proc.tasks"],
"dvc.repo.experiments.queue.tasks",
"dvc_task.proc.tasks",
],
) )
app.conf.update({"task_acks_late": True, "result_expires": None}) app.conf.update({"task_acks_late": True, "result_expires": None})
return app return app
@@ -379,10 +371,7 @@ class LocalCeleryQueue(BaseStashQueue):
raise CannotKillTasksError(remained_revs) raise CannotKillTasksError(remained_revs)
def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None: def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None:
logger.debug( logger.debug("Found active tasks: '%s' to kill", list(entries.values()))
"Found active tasks: '%s' to kill",
list(entries.values()),
)
inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks( inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks(
entries, force entries, force
) )
@@ -418,20 +407,11 @@ class LocalCeleryQueue(BaseStashQueue):
if to_kill: if to_kill:
self._kill_entries(to_kill, True) self._kill_entries(to_kill, True)
def follow( def follow(self, entry: QueueEntry, encoding: Optional[str] = None):
self,
entry: QueueEntry,
encoding: Optional[str] = None,
):
for line in self.proc.follow(entry.stash_rev, encoding): for line in self.proc.follow(entry.stash_rev, encoding):
ui.write(line, end="") ui.write(line, end="")
def logs( def logs(self, rev: str, encoding: Optional[str] = None, follow: bool = False):
self,
rev: str,
encoding: Optional[str] = None,
follow: bool = False,
):
queue_entry: Optional[QueueEntry] = self.match_queue_entry_by_name( queue_entry: Optional[QueueEntry] = self.match_queue_entry_by_name(
{rev}, self.iter_active(), self.iter_done() {rev}, self.iter_active(), self.iter_done()
).get(rev) ).get(rev)
@@ -458,8 +438,7 @@ class LocalCeleryQueue(BaseStashQueue):
f"No output logs found for experiment '{rev}'" f"No output logs found for experiment '{rev}'"
) )
with open( with open(
proc_info.stdout, proc_info.stdout, encoding=encoding or locale.getpreferredencoding()
encoding=encoding or locale.getpreferredencoding(),
) as fobj: ) as fobj:
ui.write(fobj.read()) ui.write(fobj.read())
@@ -560,9 +539,7 @@ class LocalCeleryQueue(BaseStashQueue):
return result return result
def collect_queued_data( def collect_queued_data(
self, self, baseline_revs: Optional[Collection[str]], **kwargs
baseline_revs: Optional[Collection[str]],
**kwargs,
) -> dict[str, list["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
from dvc.repo.experiments.collect import collect_rev from dvc.repo.experiments.collect import collect_rev
from dvc.repo.experiments.serialize import ( from dvc.repo.experiments.serialize import (
+3 -12
View File
@@ -37,20 +37,14 @@ def remove_tasks( # noqa: C901, PLR0912
failed_stash_revs.append(failed_rev_all[entry.stash_rev]) failed_stash_revs.append(failed_rev_all[entry.stash_rev])
try: try:
for ( for msg, queue_entry in celery_queue._iter_queued():
msg,
queue_entry,
) in celery_queue._iter_queued():
if queue_entry.stash_rev in stash_revs and msg.delivery_tag: if queue_entry.stash_rev in stash_revs and msg.delivery_tag:
celery_queue.celery.reject(msg.delivery_tag) celery_queue.celery.reject(msg.delivery_tag)
finally: finally:
celery_queue.stash.remove_revs(list(stash_revs.values())) celery_queue.stash.remove_revs(list(stash_revs.values()))
try: try:
for ( for msg, queue_entry in celery_queue._iter_processed():
msg,
queue_entry,
) in celery_queue._iter_processed():
if queue_entry not in done_entry_set: if queue_entry not in done_entry_set:
continue continue
task_id = msg.headers["id"] task_id = msg.headers["id"]
@@ -115,10 +109,7 @@ def celery_clear(
return removed return removed
def celery_remove( def celery_remove(self: "LocalCeleryQueue", revs: Collection[str]) -> list[str]:
self: "LocalCeleryQueue",
revs: Collection[str],
) -> list[str]:
"""Remove the specified entries from the queue. """Remove the specified entries from the queue.
Arguments: Arguments:
+1 -5
View File
@@ -23,11 +23,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]:
remote_url : remote executor's url remote_url : remote executor's url
""" """
refs = [] refs = []
for ref in iter_remote_refs( for ref in iter_remote_refs(scm, remote_url, base=EXPS_NAMESPACE):
scm,
remote_url,
base=EXPS_NAMESPACE,
):
if not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH: if not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH:
refs.append(ref) refs.append(ref)
return refs return refs
+1 -6
View File
@@ -182,12 +182,7 @@ class WorkspaceQueue(BaseStashQueue):
def shutdown(self, kill: bool = False): def shutdown(self, kill: bool = False):
raise NotImplementedError raise NotImplementedError
def logs( def logs(self, rev: str, encoding: Optional[str] = None, follow: bool = False):
self,
rev: str,
encoding: Optional[str] = None,
follow: bool = False,
):
raise NotImplementedError raise NotImplementedError
def get_running_exp(self) -> Optional[str]: def get_running_exp(self) -> Optional[str]:
+1 -8
View File
@@ -1,14 +1,7 @@
from collections import Counter, defaultdict from collections import Counter, defaultdict
from collections.abc import Iterable, Iterator, Mapping from collections.abc import Iterable, Iterator, Mapping
from datetime import date, datetime from datetime import date, datetime
from typing import ( from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Optional, Union
TYPE_CHECKING,
Any,
Literal,
NamedTuple,
Optional,
Union,
)
from dvc.exceptions import InvalidArgumentError from dvc.exceptions import InvalidArgumentError
from dvc.log import logger from dvc.log import logger
+2 -10
View File
@@ -118,20 +118,12 @@ class ApplyStash(Stash):
m = self.MESSAGE_RE.match(msg) m = self.MESSAGE_RE.match(msg)
if m: if m:
revs[entry.new_sha.decode("utf-8")] = ApplyStashEntry( revs[entry.new_sha.decode("utf-8")] = ApplyStashEntry(
i, i, m.group("head_rev"), m.group("rev"), m.group("name")
m.group("head_rev"),
m.group("rev"),
m.group("name"),
) )
return revs return revs
@classmethod @classmethod
def format_message( def format_message(cls, head_rev: str, rev: str, name: Optional[str] = None) -> str:
cls,
head_rev: str,
rev: str,
name: Optional[str] = None,
) -> str:
return cls.MESSAGE_FORMAT.format( return cls.MESSAGE_FORMAT.format(
head_rev=head_rev, rev=rev, name=name if name else "" head_rev=head_rev, rev=rev, name=name if name else ""
) )
+2 -12
View File
@@ -4,12 +4,7 @@ import sys
from collections import defaultdict from collections import defaultdict
from collections.abc import Generator, Iterable, Mapping from collections.abc import Generator, Iterable, Mapping
from functools import wraps from functools import wraps
from typing import ( from typing import TYPE_CHECKING, Callable, Optional, Union
TYPE_CHECKING,
Callable,
Optional,
Union,
)
from dvc.exceptions import InvalidArgumentError from dvc.exceptions import InvalidArgumentError
from dvc.repo.experiments.exceptions import AmbiguousExpRefInfo from dvc.repo.experiments.exceptions import AmbiguousExpRefInfo
@@ -282,12 +277,7 @@ def fetch_all_exps(scm: "Git", url: str, progress: Optional[Callable] = None, **
for ref in iter_remote_refs(scm, url, base=EXPS_NAMESPACE) for ref in iter_remote_refs(scm, url, base=EXPS_NAMESPACE)
if not _ignore_ref(ref) if not _ignore_ref(ref)
] ]
scm.fetch_refspecs( scm.fetch_refspecs(url, refspecs, progress=progress, **kwargs)
url,
refspecs,
progress=progress,
**kwargs,
)
def gen_random_name(): def gen_random_name():
+1 -5
View File
@@ -163,11 +163,7 @@ def fetch( # noqa: PLR0913
tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())), tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())),
) )
with ui.progress( with ui.progress(desc="Collecting", unit="entry", leave=True) as pb:
desc="Collecting",
unit="entry",
leave=True,
) as pb:
data = collect( data = collect(
[idx.data["repo"] for idx in indexes.values()], [idx.data["repo"] for idx in indexes.values()],
"remote", "remote",
+1 -6
View File
@@ -61,9 +61,4 @@ def get(
else: else:
fs = repo.dvcfs fs = repo.dvcfs
fs_path = fs.from_os_path(path) fs_path = fs.from_os_path(path)
download( download(fs, fs_path, os.path.abspath(out), jobs=jobs)
fs,
fs_path,
os.path.abspath(out),
jobs=jobs,
)
+2 -12
View File
@@ -4,14 +4,7 @@ from collections import defaultdict
from collections.abc import Iterable, Iterator from collections.abc import Iterable, Iterator
from functools import partial from functools import partial
from itertools import chain from itertools import chain
from typing import ( from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
TYPE_CHECKING,
Any,
Callable,
NamedTuple,
Optional,
Union,
)
from funcy.debug import format_time from funcy.debug import format_time
@@ -577,10 +570,7 @@ class Index:
return by_workspace return by_workspace
@staticmethod @staticmethod
def _hash_targets( def _hash_targets(targets: Iterable[Optional[str]], **kwargs: Any) -> int:
targets: Iterable[Optional[str]],
**kwargs: Any,
) -> int:
return hash( return hash(
( (
frozenset(targets), frozenset(targets),
+1 -4
View File
@@ -84,10 +84,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): # noqa:
proj = Repo(root_dir) proj = Repo(root_dir)
with proj.scm_context(autostage=True) as context: with proj.scm_context(autostage=True) as context:
files = [ files = [config.files["repo"], dvcignore]
config.files["repo"],
dvcignore,
]
ignore_file = context.scm.ignore_file ignore_file = context.scm.ignore_file
if ignore_file: if ignore_file:
files.extend([os.path.join(dvc_dir, ignore_file)]) files.extend([os.path.join(dvc_dir, ignore_file)])
+1 -7
View File
@@ -1,13 +1,7 @@
import os import os
from collections.abc import Iterable, Iterator from collections.abc import Iterable, Iterator
from itertools import chain from itertools import chain
from typing import ( from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union
TYPE_CHECKING,
Any,
Optional,
TypedDict,
Union,
)
from funcy import ldistinct from funcy import ldistinct
from scmrepo.exceptions import SCMError from scmrepo.exceptions import SCMError
+1 -5
View File
@@ -18,11 +18,7 @@ logger = logger.getChild(__name__)
@map_scm_exception() @map_scm_exception()
def _external_repo( def _external_repo(url, rev: Optional[str] = None, **kwargs) -> "Repo":
url,
rev: Optional[str] = None,
**kwargs,
) -> "Repo":
logger.debug("Creating external repo %s@%s", url, rev) logger.debug("Creating external repo %s@%s", url, rev)
path = _cached_clone(url, rev) path = _cached_clone(url, rev)
# Local HEAD points to the tip of whatever branch we first cloned from # Local HEAD points to the tip of whatever branch we first cloned from
+8 -37
View File
@@ -6,13 +6,7 @@ from collections.abc import Iterator
from copy import deepcopy from copy import deepcopy
from functools import partial from functools import partial
from multiprocessing import cpu_count from multiprocessing import cpu_count
from typing import ( from typing import TYPE_CHECKING, Any, Callable, Optional, Union
TYPE_CHECKING,
Any,
Callable,
Optional,
Union,
)
import dpath import dpath
import dpath.options import dpath.options
@@ -415,10 +409,7 @@ def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = No
onerror=onerror, onerror=onerror,
) )
dpath.merge( dpath.merge(result, {"": unpacked})
result,
{"": unpacked},
)
return result return result
@@ -458,15 +449,9 @@ def _resolve_definitions(
data_path = _normpath(fs.join(config_dir, plot_id)) data_path = _normpath(fs.join(config_dir, plot_id))
if _matches(targets, config_path, plot_id): if _matches(targets, config_path, plot_id):
unpacked = unpack_if_dir( unpacked = unpack_if_dir(
fs, fs, data_path, props={**plot_props, **props}, onerror=onerror
data_path,
props={**plot_props, **props},
onerror=onerror,
)
dpath.merge(
result,
unpacked,
) )
dpath.merge(result, unpacked)
elif _matches(targets, config_path, plot_id): elif _matches(targets, config_path, plot_id):
adjusted_props = _adjust_sources(fs, plot_props, config_dir) adjusted_props = _adjust_sources(fs, plot_props, config_dir)
dpath.merge(result, {"data": {plot_id: {**adjusted_props, **props}}}) dpath.merge(result, {"data": {plot_id: {**adjusted_props, **props}}})
@@ -489,17 +474,9 @@ def _collect_pipeline_files(repo, targets: list[str], props, onerror=None):
dvcfile_defs_dict[k] = v dvcfile_defs_dict[k] = v
resolved = _resolve_definitions( resolved = _resolve_definitions(
repo.dvcfs, repo.dvcfs, targets, props, dvcfile_path, dvcfile_defs_dict, onerror=onerror
targets,
props,
dvcfile_path,
dvcfile_defs_dict,
onerror=onerror,
)
dpath.merge(
result,
{dvcfile_path: resolved},
) )
dpath.merge(result, {dvcfile_path: resolved})
return result return result
@@ -515,15 +492,9 @@ def _collect_definitions(
props = props or {} props = props or {}
fs = repo.dvcfs fs = repo.dvcfs
dpath.merge( dpath.merge(result, _collect_pipeline_files(repo, targets, props, onerror=onerror))
result,
_collect_pipeline_files(repo, targets, props, onerror=onerror),
)
dpath.merge( dpath.merge(result, _collect_output_plots(repo, targets, props, onerror=onerror))
result,
_collect_output_plots(repo, targets, props, onerror=onerror),
)
for target in targets: for target in targets:
if not result or fs.exists(target): if not result or fs.exists(target):
+1 -5
View File
@@ -120,11 +120,7 @@ def push( # noqa: PLR0913
tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())), tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())),
) )
with ui.progress( with ui.progress(desc="Collecting", unit="entry", leave=True) as pb:
desc="Collecting",
unit="entry",
leave=True,
) as pb:
data = collect( data = collect(
[idx.data["repo"] for idx in indexes.values()], [idx.data["repo"] for idx in indexes.values()],
"remote", "remote",
+1 -9
View File
@@ -1,13 +1,5 @@
from collections.abc import Iterable from collections.abc import Iterable
from typing import ( from typing import TYPE_CHECKING, Callable, NoReturn, Optional, TypeVar, Union, cast
TYPE_CHECKING,
Callable,
NoReturn,
Optional,
TypeVar,
Union,
cast,
)
from funcy import ldistinct from funcy import ldistinct
+1 -6
View File
@@ -3,12 +3,7 @@ import shlex
from collections.abc import Iterable, Iterator from collections.abc import Iterable, Iterator
from contextlib import contextmanager from contextlib import contextmanager
from functools import wraps from functools import wraps
from typing import ( from typing import TYPE_CHECKING, Any, Optional, Union
TYPE_CHECKING,
Any,
Optional,
Union,
)
from dvc.log import logger from dvc.log import logger
from dvc.utils import relpath from dvc.utils import relpath
+1 -6
View File
@@ -402,12 +402,7 @@ class StageLoad:
from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound
try: try:
stages = self.collect( stages = self.collect(target, with_deps, recursive, graph)
target,
with_deps,
recursive,
graph,
)
except StageFileDoesNotExistError as exc: except StageFileDoesNotExistError as exc:
# collect() might try to use `target` as a stage name # collect() might try to use `target` as a stage name
# and throw error that dvc.yaml does not exist, whereas it # and throw error that dvc.yaml does not exist, whereas it
+1 -4
View File
@@ -61,10 +61,7 @@ def update( # noqa: C901
) )
if to_remote: if to_remote:
raise InvalidArgumentError("--to-remote can't be used with worktree update") raise InvalidArgumentError("--to-remote can't be used with worktree update")
update_worktree_stages( update_worktree_stages(self, other_stage_infos)
self,
other_stage_infos,
)
stages = import_stages | {stage_info.stage for stage_info in other_stage_infos} stages = import_stages | {stage_info.stage for stage_info in other_stage_infos}
return list(stages) return list(stages)
+7 -32
View File
@@ -162,20 +162,13 @@ def _merge_push_meta( # noqa: C901
out.meta.remote = remote out.meta.remote = remote
def update_worktree_stages( def update_worktree_stages(repo: "Repo", stage_infos: Iterable["StageInfo"]):
repo: "Repo",
stage_infos: Iterable["StageInfo"],
):
from dvc.repo.index import IndexView from dvc.repo.index import IndexView
def outs_filter(out: "Output") -> bool: def outs_filter(out: "Output") -> bool:
return out.is_in_repo and out.use_cache and out.can_push return out.is_in_repo and out.use_cache and out.can_push
view = IndexView( view = IndexView(repo.index, stage_infos, outs_filter=outs_filter)
repo.index,
stage_infos,
outs_filter=outs_filter,
)
local_index = view.data["repo"] local_index = view.data["repo"]
remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {} remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {}
for stage in view.stages: for stage in view.stages:
@@ -194,10 +187,7 @@ def _update_worktree_out(
remote_name = out.remote or out.meta.remote remote_name = out.remote or out.meta.remote
if not remote_name: if not remote_name:
logger.warning( logger.warning("Could not update '%s', it was never pushed to a remote", out)
"Could not update '%s', it was never pushed to a remote",
out,
)
return return
if remote_name in remote_indexes: if remote_name in remote_indexes:
@@ -211,10 +201,7 @@ def _update_worktree_out(
remote_indexes[remote_name] = remote, remote_index remote_indexes[remote_name] = remote, remote_index
_workspace, key = out.index_key _workspace, key = out.index_key
if key not in remote_index: if key not in remote_index:
logger.warning( logger.warning("Could not update '%s', it does not exist in the remote", out)
"Could not update '%s', it does not exist in the remote",
out,
)
return return
entry = remote_index[key] entry = remote_index[key]
@@ -226,20 +213,11 @@ def _update_worktree_out(
for subkey, subentry in remote_index.iteritems(key) for subkey, subentry in remote_index.iteritems(key)
) )
): ):
logger.warning( logger.warning("Could not update '%s', directory is empty in the remote", out)
"Could not update '%s', directory is empty in the remote",
out,
)
return return
_fetch_out_changes(out, local_index, remote_index, remote) _fetch_out_changes(out, local_index, remote_index, remote)
_update_out_meta( _update_out_meta(repo, out, local_index, remote_index, remote)
repo,
out,
local_index,
remote_index,
remote,
)
def _fetch_out_changes( def _fetch_out_changes(
@@ -253,10 +231,7 @@ def _fetch_out_changes(
old, new = _get_diff_indexes(out, local_index, remote_index) old, new = _get_diff_indexes(out, local_index, remote_index)
with TqdmCallback( with TqdmCallback(unit="entry", desc="Comparing indexes") as cb:
unit="entry",
desc="Comparing indexes",
) as cb:
diff = compare( diff = compare(
old, old,
new, new,
+1 -7
View File
@@ -3,13 +3,7 @@ import os
from collections.abc import Iterator, Mapping from collections.abc import Iterator, Mapping
from contextlib import contextmanager from contextlib import contextmanager
from functools import partial from functools import partial
from typing import ( from typing import TYPE_CHECKING, Literal, Optional, Union, overload
TYPE_CHECKING,
Literal,
Optional,
Union,
overload,
)
from funcy import group_by from funcy import group_by
from scmrepo.base import Base # noqa: F401 from scmrepo.base import Base # noqa: F401
+2 -12
View File
@@ -4,13 +4,7 @@ from collections import defaultdict
from collections.abc import Iterable from collections.abc import Iterable
from contextlib import suppress from contextlib import suppress
from dataclasses import dataclass from dataclasses import dataclass
from typing import ( from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
TYPE_CHECKING,
Any,
Optional,
TypeVar,
Union,
)
from funcy import project from funcy import project
@@ -664,11 +658,7 @@ class Stage(params.StageParams):
return stats return stats
for out in self.filter_outs(kwargs.get("filter_info")): for out in self.filter_outs(kwargs.get("filter_info")):
key, outs = self._checkout( key, outs = self._checkout(out, allow_missing=allow_missing, **kwargs)
out,
allow_missing=allow_missing,
**kwargs,
)
if key: if key:
stats[key].extend(outs) stats[key].extend(outs)
return stats return stats
+1 -4
View File
@@ -266,10 +266,7 @@ class StageCache:
src_name = from_fs.name(src) src_name = from_fs.name(src)
parent_name = from_fs.name(from_fs.parent(src)) parent_name = from_fs.name(from_fs.parent(src))
with TqdmCallback( with TqdmCallback(desc=src_name, bytes=True) as cb:
desc=src_name,
bytes=True,
) as cb:
func(from_fs, src, to_fs, dst, callback=cb) func(from_fs, src, to_fs, dst, callback=cb)
ret.append((parent_name, src_name)) ret.append((parent_name, src_name))
return ret return ret
+2 -11
View File
@@ -44,13 +44,7 @@ def update_import(
stage.frozen = frozen stage.frozen = frozen
def sync_import( def sync_import(stage, dry=False, force=False, jobs=None, no_download=False):
stage,
dry=False,
force=False,
jobs=None,
no_download=False,
):
"""Synchronize import's outs to the workspace.""" """Synchronize import's outs to the workspace."""
logger.info("Importing '%s' -> '%s'", stage.deps[0], stage.outs[0]) logger.info("Importing '%s' -> '%s'", stage.deps[0], stage.outs[0])
if dry: if dry:
@@ -64,7 +58,4 @@ def sync_import(
if stage.is_repo_import or stage.is_db_import: if stage.is_repo_import or stage.is_db_import:
stage.deps[0].update() stage.deps[0].update()
else: else:
stage.deps[0].download( stage.deps[0].download(stage.outs[0], jobs=jobs)
stage.outs[0],
jobs=jobs,
)
+2 -9
View File
@@ -25,12 +25,7 @@ logger = logger.getChild(__name__)
class StageLoader(Mapping): class StageLoader(Mapping):
def __init__( def __init__(self, dvcfile: "ProjectFile", data, lockfile_data=None):
self,
dvcfile: "ProjectFile",
data,
lockfile_data=None,
):
self.dvcfile = dvcfile self.dvcfile = dvcfile
self.resolver = self.dvcfile.resolver self.resolver = self.dvcfile.resolver
self.data = data or {} self.data = data or {}
@@ -190,9 +185,7 @@ class SingleStageLoader(Mapping):
def __getitem__(self, item): def __getitem__(self, item):
if item: if item:
logger.warning( logger.warning(
"Ignoring name '%s' for single stage in '%s'.", "Ignoring name '%s' for single stage in '%s'.", item, self.dvcfile
item,
self.dvcfile,
) )
# during `load`, we remove attributes from stage data, so as to # during `load`, we remove attributes from stage data, so as to
# not duplicate, therefore, for MappingView, we need to deepcopy. # not duplicate, therefore, for MappingView, we need to deepcopy.
+1 -7
View File
@@ -119,13 +119,7 @@ def _pull_missing_deps(stage):
stage.repo.pull(dep.def_path) stage.repo.pull(dep.def_path)
def run_stage( def run_stage(stage, dry=False, force=False, run_env=None, **kwargs):
stage,
dry=False,
force=False,
run_env=None,
**kwargs,
):
if not force: if not force:
if kwargs.get("pull") and not dry: if kwargs.get("pull") and not dry:
_pull_missing_deps(stage) _pull_missing_deps(stage)
+1 -7
View File
@@ -1,13 +1,7 @@
from collections import OrderedDict from collections import OrderedDict
from collections.abc import Iterable from collections.abc import Iterable
from operator import attrgetter from operator import attrgetter
from typing import ( from typing import TYPE_CHECKING, Any, Optional, Union, no_type_check
TYPE_CHECKING,
Any,
Optional,
Union,
no_type_check,
)
from funcy import post_processing from funcy import post_processing
+1 -6
View File
@@ -15,12 +15,7 @@ class TestAPI:
assert api.get_url("foo") == expected_url assert api.get_url("foo") == expected_url
def test_open(self, tmp_dir, dvc, remote): def test_open(self, tmp_dir, dvc, remote):
tmp_dir.dvc_gen( tmp_dir.dvc_gen({"foo": "foo-text", "dir": {"bar": "bar-text"}})
{
"foo": "foo-text",
"dir": {"bar": "bar-text"},
}
)
dvc.push() dvc.push()
# Remove cache to force download # Remove cache to force download
@@ -5,13 +5,7 @@ import pytest
pytestmark = pytest.mark.requires(minversion=(2, 15, 0), reason="new command") pytestmark = pytest.mark.requires(minversion=(2, 15, 0), reason="new command")
def test_data_status( def test_data_status(bench_dvc, tmp_dir, scm, dvc, make_dataset):
bench_dvc,
tmp_dir,
scm,
dvc,
make_dataset,
):
args = ("data", "status") args = ("data", "status")
dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False) dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
rmtree(dvc.tmp_dir) rmtree(dvc.tmp_dir)
+1 -5
View File
@@ -86,11 +86,7 @@ def pytest_addoption(parser):
help="Path or url to dvc-bench git repo (for loading benchmark datasets)", help="Path or url to dvc-bench git repo (for loading benchmark datasets)",
) )
parser.addoption( parser.addoption("--project-rev", type=str, help="Project revision to test")
"--project-rev",
type=str,
help="Project revision to test",
)
parser.addoption( parser.addoption(
"--project-git-repo", "--project-git-repo",
+1 -5
View File
@@ -246,11 +246,7 @@ def docker_services(
try: try:
cmd = "docker-compose version" cmd = "docker-compose version"
subprocess.check_output( subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) # noqa: S602
cmd,
stderr=subprocess.STDOUT,
shell=True, # noqa: S602
)
except subprocess.CalledProcessError as err: except subprocess.CalledProcessError as err:
out = (err.output or b"").decode("utf-8") out = (err.output or b"").decode("utf-8")
pytest.skip(f"docker-compose is not installed: {out}") pytest.skip(f"docker-compose is not installed: {out}")
+1 -3
View File
@@ -209,9 +209,7 @@ class URLInfo(_BasePath):
return self._spath return self._spath
@cached_property @cached_property
def _path( def _path(self) -> "_URLPathInfo":
self,
) -> "_URLPathInfo":
return _URLPathInfo(self._spath) return _URLPathInfo(self._spath)
@property @property
+1 -3
View File
@@ -20,7 +20,5 @@ def _add_script(tmp_dir, path, contents=""):
@pytest.fixture @pytest.fixture
def copy_script( def copy_script(tmp_dir):
tmp_dir,
):
return _add_script(tmp_dir, "copy.py", COPY_SCRIPT) return _add_script(tmp_dir, "copy.py", COPY_SCRIPT)
+2 -8
View File
@@ -173,9 +173,7 @@ class TestImportURLVersionAware:
dvc.cache.local.clear() dvc.cache.local.clear()
remove(tmp_dir / "data_dir") remove(tmp_dir / "data_dir")
assert dvc.pull()["fetched"] == 1 assert dvc.pull()["fetched"] == 1
assert (tmp_dir / "data_dir").read_text() == { assert (tmp_dir / "data_dir").read_text() == {"subdir": {"file": "file"}}
"subdir": {"file": "file"},
}
dvc.commit(force=True) dvc.commit(force=True)
assert dvc.status() == {} assert dvc.status() == {}
@@ -193,11 +191,7 @@ class TestLsUrl:
cloud.gen({fname: "foo contents"}) cloud.gen({fname: "foo contents"})
fs, fs_path = parse_external_url(cloud.url, cloud.config) fs, fs_path = parse_external_url(cloud.url, cloud.config)
result = ls_url(str(cloud / fname), fs_config=cloud.config) result = ls_url(str(cloud / fname), fs_config=cloud.config)
match_files( match_files(fs, result, [{"path": fs.join(fs_path, fname), "isdir": False}])
fs,
result,
[{"path": fs.join(fs_path, fname), "isdir": False}],
)
def test_dir(self, cloud): def test_dir(self, cloud):
cloud.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"}) cloud.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"})
+1 -8
View File
@@ -1,13 +1,6 @@
from collections.abc import Iterable, Iterator, Sequence from collections.abc import Iterable, Iterator, Sequence
from contextlib import contextmanager, nullcontext from contextlib import contextmanager, nullcontext
from typing import ( from typing import TYPE_CHECKING, Any, Callable, Optional, TextIO, Union
TYPE_CHECKING,
Any,
Callable,
Optional,
TextIO,
Union,
)
import colorama import colorama
+1 -3
View File
@@ -17,9 +17,7 @@ def parse_params(path_params: Iterable[str]) -> list[dict[str, list[str]]]:
return [{path: params} for path, params in ret.items()] return [{path: params} for path, params in ret.items()]
def to_path_overrides( def to_path_overrides(path_params: Iterable[str]) -> dict[str, list[str]]:
path_params: Iterable[str],
) -> dict[str, list[str]]:
"""Group overrides by path""" """Group overrides by path"""
from dvc.dependency.param import ParamsDependency from dvc.dependency.param import ParamsDependency
+1 -9
View File
@@ -1,15 +1,7 @@
"""Common utilities for serialize.""" """Common utilities for serialize."""
import os import os
from contextlib import AbstractContextManager, contextmanager from contextlib import AbstractContextManager, contextmanager
from typing import ( from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol, TextIO, Union
TYPE_CHECKING,
Any,
Callable,
Optional,
Protocol,
TextIO,
Union,
)
from funcy import reraise from funcy import reraise
+9 -45
View File
@@ -31,38 +31,15 @@ def params_repo(tmp_dir, scm, dvc):
tmp_dir.gen("params.json", '{"bar": 2, "foobar": 3}') tmp_dir.gen("params.json", '{"bar": 2, "foobar": 3}')
tmp_dir.gen("other_params.json", '{"foo": {"bar": 4}}') tmp_dir.gen("other_params.json", '{"foo": {"bar": 4}}')
dvc.run( dvc.run(name="stage-0", cmd="echo stage-0")
name="stage-0",
cmd="echo stage-0",
)
dvc.run( dvc.run(name="stage-1", cmd="echo stage-1", params=["foo", "params.json:bar"])
name="stage-1",
cmd="echo stage-1",
params=["foo", "params.json:bar"],
)
dvc.run( dvc.run(name="stage-2", cmd="echo stage-2", params=["other_params.json:foo"])
name="stage-2",
cmd="echo stage-2",
params=["other_params.json:foo"],
)
dvc.run( dvc.run(name="stage-3", cmd="echo stage-2", params=["params.json:foobar"])
name="stage-3",
cmd="echo stage-2",
params=["params.json:foobar"],
)
scm.add( scm.add(["params.yaml", "params.json", "other_params.json", "dvc.yaml", "dvc.lock"])
[
"params.yaml",
"params.json",
"other_params.json",
"dvc.yaml",
"dvc.lock",
]
)
scm.commit("commit dvc files") scm.commit("commit dvc files")
tmp_dir.gen("params.yaml", "foo: 5") tmp_dir.gen("params.yaml", "foo: 5")
@@ -77,10 +54,7 @@ def metrics_repo(tmp_dir, scm, dvc, run_copy_metrics):
scm.commit("prepare data") scm.commit("prepare data")
sub_dir = tmp_dir / "eval" sub_dir = tmp_dir / "eval"
sub_dir.mkdir() sub_dir.mkdir()
tmp_dir.gen( tmp_dir.gen("tmp_train_val_metrics.json", json.dumps(TRAIN_METRICS[0]))
"tmp_train_val_metrics.json",
json.dumps(TRAIN_METRICS[0]),
)
train_metrics_file = os.path.join(sub_dir, "train_val_metrics.json") train_metrics_file = os.path.join(sub_dir, "train_val_metrics.json")
run_copy_metrics( run_copy_metrics(
"tmp_train_val_metrics.json", "tmp_train_val_metrics.json",
@@ -107,10 +81,7 @@ def metrics_repo(tmp_dir, scm, dvc, run_copy_metrics):
scm.commit("test model") scm.commit("test model")
with tmp_dir.branch("better-model", new=True): with tmp_dir.branch("better-model", new=True):
tmp_dir.gen( tmp_dir.gen("tmp_train_val_metrics.json", json.dumps(TRAIN_METRICS[1]))
"tmp_train_val_metrics.json",
json.dumps(TRAIN_METRICS[1]),
)
run_copy_metrics( run_copy_metrics(
"tmp_train_val_metrics.json", "tmp_train_val_metrics.json",
train_metrics_file, train_metrics_file,
@@ -243,11 +214,7 @@ def test_params_show_while_running_stage(tmp_dir, dvc):
def test_params_show_repo(tmp_dir, erepo_dir): def test_params_show_repo(tmp_dir, erepo_dir):
with erepo_dir.chdir(): with erepo_dir.chdir():
erepo_dir.scm_gen("params.yaml", "foo: 1", commit="Create params.yaml") erepo_dir.scm_gen("params.yaml", "foo: 1", commit="Create params.yaml")
erepo_dir.dvc.run( erepo_dir.dvc.run(name="stage-1", cmd="echo stage-1", params=["foo"])
name="stage-1",
cmd="echo stage-1",
params=["foo"],
)
assert api.params_show(repo=erepo_dir) == {"foo": 1} assert api.params_show(repo=erepo_dir) == {"foo": 1}
@@ -268,10 +235,7 @@ def test_params_show_no_params_found(tmp_dir, dvc):
def test_params_show_stage_without_params(tmp_dir, dvc): def test_params_show_stage_without_params(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "foo: 1") tmp_dir.gen("params.yaml", "foo: 1")
dvc.run( dvc.run(name="stage-0", cmd="echo stage-0")
name="stage-0",
cmd="echo stage-0",
)
assert api.params_show(stages="stage-0") == {} assert api.params_show(stages="stage-0") == {}
+3 -18
View File
@@ -70,9 +70,7 @@ def test_artifacts_add_subdir(tmp_dir, dvc):
name: Artifact(**values) for name, values in dvcyaml["artifacts"].items() name: Artifact(**values) for name, values in dvcyaml["artifacts"].items()
} }
artifacts["new"] = new_art artifacts["new"] = new_art
assert tmp_dir.dvc.artifacts.read() == { assert tmp_dir.dvc.artifacts.read() == {f"subdir{os.path.sep}dvc.yaml": artifacts}
f"subdir{os.path.sep}dvc.yaml": artifacts,
}
def test_artifacts_add_abspath(tmp_dir, dvc): def test_artifacts_add_abspath(tmp_dir, dvc):
@@ -114,11 +112,7 @@ bad_dvcyaml_extra_field = {
} }
bad_dvcyaml_missing_path = { bad_dvcyaml_missing_path = {"artifacts": {"lol": {}}}
"artifacts": {
"lol": {},
}
}
@pytest.mark.parametrize( @pytest.mark.parametrize(
@@ -148,16 +142,7 @@ def test_artifacts_read_fails_on_id_duplication(tmp_dir, dvc):
@pytest.mark.parametrize( @pytest.mark.parametrize(
"name", "name", ["1", "m", "nn", "m1", "1nn", "model-prod", "model-prod-v1"]
[
"1",
"m",
"nn",
"m1",
"1nn",
"model-prod",
"model-prod-v1",
],
) )
def test_name_is_compatible(name): def test_name_is_compatible(name):
check_name_format(name) check_name_format(name)
+1 -5
View File
@@ -34,10 +34,6 @@ def params_repo(tmp_dir, scm, dvc):
(tmp_dir / "params.yaml").dump( (tmp_dir / "params.yaml").dump(
{"foo": [{"bar": 1}, {"baz": 2}], "goo": {"bag": 3.0}, "lorem": False} {"foo": [{"bar": 1}, {"baz": 2}], "goo": {"bag": 3.0}, "lorem": False}
) )
dvc.run( dvc.run(cmd="echo foo", params=["params.yaml:"], name="foo")
cmd="echo foo",
params=["params.yaml:"],
name="foo",
)
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml"]) scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml"])
scm.commit("init") scm.commit("init")
+1 -5
View File
@@ -46,11 +46,7 @@ def test_apply_failed(tmp_dir, scm, dvc, failed_exp_stage, mocker):
), ),
], ],
) )
mocker.patch.object( mocker.patch.object(queue, "iter_queued", return_value=[])
queue,
"iter_queued",
return_value=[],
)
dvc.experiments.apply(exp_rev) dvc.experiments.apply(exp_rev)
assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 3" assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 3"
+8 -41
View File
@@ -100,14 +100,7 @@ def test_file_permissions(tmp_dir, scm, dvc, exp_stage, mocker):
assert stat.S_IMODE(os.stat(tmp_dir / "copy.py").st_mode) == mode assert stat.S_IMODE(os.stat(tmp_dir / "copy.py").st_mode) == mode
def test_failed_exp_workspace( def test_failed_exp_workspace(tmp_dir, scm, dvc, failed_exp_stage, mocker, capsys):
tmp_dir,
scm,
dvc,
failed_exp_stage,
mocker,
capsys,
):
tmp_dir.gen("params.yaml", "foo: 2") tmp_dir.gen("params.yaml", "foo: 2")
with pytest.raises(ReproductionError): with pytest.raises(ReproductionError):
dvc.experiments.run(failed_exp_stage.addressing) dvc.experiments.run(failed_exp_stage.addressing)
@@ -445,13 +438,7 @@ def test_subrepo(tmp_dir, request, scm, workspace):
name="copy-file", name="copy-file",
no_exec=True, no_exec=True,
) )
scm.add( scm.add([subrepo / "dvc.yaml", subrepo / "copy.py", subrepo / "params.yaml"])
[
subrepo / "dvc.yaml",
subrepo / "copy.py",
subrepo / "params.yaml",
]
)
scm.commit("init") scm.commit("init")
results = subrepo.dvc.experiments.run( results = subrepo.dvc.experiments.run(
@@ -593,11 +580,7 @@ def test_experiment_name_invalid(tmp_dir, scm, dvc, exp_stage, mocker):
new_mock = mocker.spy(BaseStashQueue, "_stash_exp") new_mock = mocker.spy(BaseStashQueue, "_stash_exp")
with pytest.raises(InvalidArgumentError): with pytest.raises(InvalidArgumentError):
dvc.experiments.run( dvc.experiments.run(exp_stage.addressing, name="fo^o", params=["foo=3"])
exp_stage.addressing,
name="fo^o",
params=["foo=3"],
)
new_mock.assert_not_called() new_mock.assert_not_called()
@@ -651,10 +634,7 @@ def test_run_env(tmp_dir, dvc, scm, mocker):
) )
(tmp_dir / "dump_run_env.py").write_text(dump_run_env) (tmp_dir / "dump_run_env.py").write_text(dump_run_env)
baseline = scm.get_rev() baseline = scm.get_rev()
dvc.stage.add( dvc.stage.add(cmd="python dump_run_env.py", name="run_env")
cmd="python dump_run_env.py",
name="run_env",
)
dvc.experiments.run() dvc.experiments.run()
assert (tmp_dir / DVC_EXP_BASELINE_REV).read_text().strip() == baseline assert (tmp_dir / DVC_EXP_BASELINE_REV).read_text().strip() == baseline
assert (tmp_dir / DVC_EXP_NAME).read_text().strip() assert (tmp_dir / DVC_EXP_NAME).read_text().strip()
@@ -728,11 +708,7 @@ def test_local_config_is_propagated_to_tmp(tmp_dir, scm, dvc):
@pytest.mark.parametrize("tmp", [True, False]) @pytest.mark.parametrize("tmp", [True, False])
def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp): def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
(tmp_dir / "dvc.yaml").dump( (tmp_dir / "dvc.yaml").dump(
{ {"metrics": ["metrics.json"], "params": ["params.yaml"], "plots": ["plots.csv"]}
"metrics": ["metrics.json"],
"params": ["params.yaml"],
"plots": ["plots.csv"],
}
) )
stage = dvc.stage.add( stage = dvc.stage.add(
cmd="touch metrics.json && touch params.yaml && touch plots.csv", cmd="touch metrics.json && touch params.yaml && touch plots.csv",
@@ -748,10 +724,7 @@ def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
@pytest.mark.parametrize("tmp", [True, False]) @pytest.mark.parametrize("tmp", [True, False])
def test_copy_paths(tmp_dir, scm, dvc, tmp): def test_copy_paths(tmp_dir, scm, dvc, tmp):
stage = dvc.stage.add( stage = dvc.stage.add(cmd="cat file && ls dir", name="foo")
cmd="cat file && ls dir",
name="foo",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml") scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
(tmp_dir / "dir").mkdir() (tmp_dir / "dir").mkdir()
@@ -770,10 +743,7 @@ def test_copy_paths(tmp_dir, scm, dvc, tmp):
def test_copy_paths_errors(tmp_dir, scm, dvc, mocker): def test_copy_paths_errors(tmp_dir, scm, dvc, mocker):
stage = dvc.stage.add( stage = dvc.stage.add(cmd="echo foo", name="foo")
cmd="echo foo",
name="foo",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml") scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
with pytest.raises(DvcException, match="Unable to copy"): with pytest.raises(DvcException, match="Unable to copy"):
@@ -814,10 +784,7 @@ def test_mixed_git_dvc_out(tmp_dir, scm, dvc, exp_stage):
@pytest.mark.parametrize("tmp", [True, False]) @pytest.mark.parametrize("tmp", [True, False])
def test_custom_commit_message(tmp_dir, scm, dvc, tmp): def test_custom_commit_message(tmp_dir, scm, dvc, tmp):
stage = dvc.stage.add( stage = dvc.stage.add(cmd="echo foo", name="foo")
cmd="echo foo",
name="foo",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml") scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
exp = first( exp = first(
+4 -24
View File
@@ -10,15 +10,7 @@ def to_dict(tasks):
@pytest.mark.parametrize("follow", [True, False]) @pytest.mark.parametrize("follow", [True, False])
def test_celery_logs( def test_celery_logs(tmp_dir, scm, dvc, failed_exp_stage, follow, capsys, test_queue):
tmp_dir,
scm,
dvc,
failed_exp_stage,
follow,
capsys,
test_queue,
):
celery_queue = dvc.experiments.celery_queue celery_queue = dvc.experiments.celery_queue
dvc.experiments.run(failed_exp_stage.addressing, queue=True, name="foo") dvc.experiments.run(failed_exp_stage.addressing, queue=True, name="foo")
dvc.experiments.run(run_all=True) dvc.experiments.run(run_all=True)
@@ -37,23 +29,14 @@ def test_queue_doesnt_remove_untracked_params_file(tmp_dir, dvc, scm):
"""Regression test for https://github.com/iterative/dvc/issues/7842""" """Regression test for https://github.com/iterative/dvc/issues/7842"""
tmp_dir.gen("params.yaml", "foo: 1") tmp_dir.gen("params.yaml", "foo: 1")
stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo") stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
scm.add( scm.add(["dvc.yaml", "dvc.lock", ".gitignore"])
[
"dvc.yaml",
"dvc.lock",
".gitignore",
]
)
scm.commit("init") scm.commit("init")
dvc.experiments.run(stage.addressing, params=["foo=2"], queue=True) dvc.experiments.run(stage.addressing, params=["foo=2"], queue=True)
assert (tmp_dir / "params.yaml").exists() assert (tmp_dir / "params.yaml").exists()
def test_copy_paths_queue(tmp_dir, scm, dvc): def test_copy_paths_queue(tmp_dir, scm, dvc):
stage = dvc.stage.add( stage = dvc.stage.add(cmd="cat file && ls dir", name="foo")
cmd="cat file && ls dir",
name="foo",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml") scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
(tmp_dir / "dir").mkdir() (tmp_dir / "dir").mkdir()
@@ -72,10 +55,7 @@ def test_copy_paths_queue(tmp_dir, scm, dvc):
def test_custom_commit_message_queue(tmp_dir, scm, dvc): def test_custom_commit_message_queue(tmp_dir, scm, dvc):
stage = dvc.stage.add( stage = dvc.stage.add(cmd="echo foo", name="foo")
cmd="echo foo",
name="foo",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml") scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
dvc.experiments.run(stage.addressing, queue=True, message="custom commit message") dvc.experiments.run(stage.addressing, queue=True, message="custom commit message")
+1 -5
View File
@@ -356,11 +356,7 @@ def test_get(tmp_dir, scm, dvc, exp_stage, erepo_dir, use_ref):
exp_ref = first(exp_refs_by_rev(scm, exp_rev)) exp_ref = first(exp_refs_by_rev(scm, exp_rev))
with erepo_dir.chdir(): with erepo_dir.chdir():
Repo.get( Repo.get(str(tmp_dir), "params.yaml", rev=exp_ref.name if use_ref else exp_rev)
str(tmp_dir),
"params.yaml",
rev=exp_ref.name if use_ref else exp_rev,
)
assert (erepo_dir / "params.yaml").read_text().strip() == "foo: 2" assert (erepo_dir / "params.yaml").read_text().strip() == "foo: 2"
+2 -9
View File
@@ -121,11 +121,7 @@ def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc): def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
(tmp_dir / "dvc.yaml").dump( (tmp_dir / "dvc.yaml").dump(
{ {"metrics": ["metrics.json"], "params": ["params.yaml"], "plots": ["plots.csv"]}
"metrics": ["metrics.json"],
"params": ["params.yaml"],
"plots": ["plots.csv"],
}
) )
stage = dvc.stage.add( stage = dvc.stage.add(
cmd="touch metrics.json && touch params.yaml && touch plots.csv", cmd="touch metrics.json && touch params.yaml && touch plots.csv",
@@ -140,10 +136,7 @@ def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
def test_untracked_dvclock_is_included_in_exp(tmp_dir, scm, dvc): def test_untracked_dvclock_is_included_in_exp(tmp_dir, scm, dvc):
stage = dvc.stage.add( stage = dvc.stage.add(cmd="echo foo", name="foo")
cmd="echo foo",
name="foo",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml") scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
dvc.reproduce(stage.addressing) dvc.reproduce(stage.addressing)
+1 -5
View File
@@ -103,11 +103,7 @@ def test_hydra_sweep(
patched = mocker.patch.object(dvc.experiments, "queue_one") patched = mocker.patch.object(dvc.experiments, "queue_one")
if hydra_enabled: if hydra_enabled:
hydra_setup( hydra_setup(tmp_dir, config_dir="conf", config_name="config")
tmp_dir,
config_dir="conf",
config_name="config",
)
with dvc.config.edit() as conf: with dvc.config.edit() as conf:
conf["hydra"]["enabled"] = True conf["hydra"]["enabled"] = True
+9 -47
View File
@@ -49,11 +49,7 @@ def make_executor(local=None, **kwargs):
local_executor.update(local) local_executor.update(local)
else: else:
local_executor = ANY local_executor = ANY
data = { data = {"state": ANY, "local": local_executor, "name": ANY}
"state": ANY,
"local": local_executor,
"name": ANY,
}
data.update(kwargs) data.update(kwargs)
return data return data
@@ -62,13 +58,7 @@ def make_data(params=None, **kwargs):
params = {"data": params or {"foo": 1}} params = {"data": params or {"foo": 1}}
data = { data = {
"rev": ANY, "rev": ANY,
"deps": { "deps": {"copy.py": {"hash": ANY, "size": ANY, "nfiles": None}},
"copy.py": {
"hash": ANY,
"size": ANY,
"nfiles": None,
}
},
"metrics": {"metrics.yaml": params}, "metrics": {"metrics.yaml": params},
"outs": {}, "outs": {},
"params": {"params.yaml": params}, "params": {"params.yaml": params},
@@ -233,23 +223,14 @@ def test_show_failed_experiment(tmp_dir, scm, dvc, failed_exp_stage, test_queue)
"experiments": None, "experiments": None,
} }
], ],
"executor": make_executor( "executor": make_executor(state="failed", local={"returncode": 255}),
state="failed",
local={"returncode": 255},
),
"name": ANY, "name": ANY,
} }
], ],
} }
def test_show_filter( def test_show_filter(tmp_dir, scm, dvc, capsys, copy_script):
tmp_dir,
scm,
dvc,
capsys,
copy_script,
):
capsys.readouterr() capsys.readouterr()
params_file = tmp_dir / "params.yaml" params_file = tmp_dir / "params.yaml"
@@ -487,10 +468,7 @@ def test_show_csv(tmp_dir, scm, dvc, exp_stage, capsys):
def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script): def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script):
params_file = tmp_dir / "params.yaml" params_file = tmp_dir / "params.yaml"
params_data = { params_data = {"foo": 1, "goobar": 1}
"foo": 1,
"goobar": 1,
}
(tmp_dir / params_file).dump(params_data) (tmp_dir / params_file).dump(params_data)
dvc.run( dvc.run(
@@ -534,10 +512,7 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script):
@pytest.mark.vscode @pytest.mark.vscode
def test_show_outs(tmp_dir, dvc, scm, erepo_dir, copy_script): def test_show_outs(tmp_dir, dvc, scm, erepo_dir, copy_script):
params_file = tmp_dir / "params.yaml" params_file = tmp_dir / "params.yaml"
params_data = { params_data = {"foo": 1, "bar": 1}
"foo": 1,
"bar": 1,
}
(tmp_dir / params_file).dump(params_data) (tmp_dir / params_file).dump(params_data)
dvc.run( dvc.run(
@@ -641,9 +616,7 @@ def test_show_outs(tmp_dir, dvc, scm, erepo_dir, copy_script):
def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script): def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script):
params_file = tmp_dir / "params.yaml" params_file = tmp_dir / "params.yaml"
params_data = { params_data = {"foo": 1}
"foo": 1,
}
(tmp_dir / params_file).dump(params_data) (tmp_dir / params_file).dump(params_data)
dvc.run( dvc.run(
@@ -674,14 +647,7 @@ def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script):
name="copy-file", name="copy-file",
deps=["copy.py"], deps=["copy.py"],
) )
scm.add( scm.add(["dvc.yaml", "dvc.lock", "params.yaml", "scores.yaml"])
[
"dvc.yaml",
"dvc.lock",
"params.yaml",
"scores.yaml",
]
)
scm.commit("scores.yaml") scm.commit("scores.yaml")
scores_rev = scm.get_rev() scores_rev = scm.get_rev()
@@ -709,11 +675,7 @@ def test_show_sorted_deps(tmp_dir, dvc, scm, capsys):
tmp_dir.gen("c", "c") tmp_dir.gen("c", "c")
tmp_dir.gen("z", "z") tmp_dir.gen("z", "z")
dvc.run( dvc.run(cmd="echo foo", name="deps", deps=["a", "b", "z", "c"])
cmd="echo foo",
name="deps",
deps=["a", "b", "z", "c"],
)
capsys.readouterr() capsys.readouterr()
assert main(["exp", "show", "--csv"]) == 0 assert main(["exp", "show", "--csv"]) == 0
+3 -13
View File
@@ -9,11 +9,7 @@ from dvc.exceptions import ReproductionError
@pytest.mark.parametrize("staged", [True, False]) @pytest.mark.parametrize("staged", [True, False])
def test_deleted(tmp_dir, scm, dvc, tmp, staged): def test_deleted(tmp_dir, scm, dvc, tmp, staged):
tmp_dir.scm_gen("file", "file", commit="commit file") tmp_dir.scm_gen("file", "file", commit="commit file")
stage = dvc.stage.add( stage = dvc.stage.add(cmd="cat file", deps=["file"], name="foo")
cmd="cat file",
deps=["file"],
name="foo",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml") scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
file = tmp_dir / "file" file = tmp_dir / "file"
@@ -33,10 +29,7 @@ def test_deleted(tmp_dir, scm, dvc, tmp, staged):
@pytest.mark.parametrize("staged", [True, False]) @pytest.mark.parametrize("staged", [True, False])
def test_modified(tmp_dir, scm, dvc, caplog, tmp, staged): def test_modified(tmp_dir, scm, dvc, caplog, tmp, staged):
tmp_dir.scm_gen("file", "file", commit="commit file") tmp_dir.scm_gen("file", "file", commit="commit file")
stage = dvc.stage.add( stage = dvc.stage.add(cmd="cat file", name="foo")
cmd="cat file",
name="foo",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml") scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
(tmp_dir / "file").write_text("modified_file") (tmp_dir / "file").write_text("modified_file")
@@ -52,10 +45,7 @@ def test_modified(tmp_dir, scm, dvc, caplog, tmp, staged):
@pytest.mark.parametrize("tmp", [True, False]) @pytest.mark.parametrize("tmp", [True, False])
def test_staged_new_file(tmp_dir, scm, dvc, tmp): def test_staged_new_file(tmp_dir, scm, dvc, tmp):
stage = dvc.stage.add( stage = dvc.stage.add(cmd="cat file", name="foo")
cmd="cat file",
name="foo",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml") scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
(tmp_dir / "file").write_text("file") (tmp_dir / "file").write_text("file")
+1 -10
View File
@@ -3,16 +3,7 @@ from funcy import first
def test_generate_random_exp_name(tmp_dir, dvc, scm, exp_stage, mocker): def test_generate_random_exp_name(tmp_dir, dvc, scm, exp_stage, mocker):
mocked_generator = mocker.MagicMock() mocked_generator = mocker.MagicMock()
mocked_generator.choice.side_effect = [ mocked_generator.choice.side_effect = [0, 0, 0, 0, 1, 1, 0, 0]
0,
0,
0,
0,
1,
1,
0,
0,
]
mocker.patch( mocker.patch(
"dvc.repo.experiments.utils.random.Random", return_value=mocked_generator "dvc.repo.experiments.utils.random.Random", return_value=mocked_generator
) )
+2 -11
View File
@@ -160,13 +160,7 @@ def test_missing_cache(tmp_dir, dvc, run_copy_metrics):
result = dvc.metrics.show() result = dvc.metrics.show()
metrics2 = result[""]["data"].pop("metrics2.yaml") metrics2 = result[""]["data"].pop("metrics2.yaml")
assert isinstance(metrics2["error"], FileNotFoundError) assert isinstance(metrics2["error"], FileNotFoundError)
assert result == { assert result == {"": {"data": {"metrics.yaml": {"data": 1.1}}}}
"": {
"data": {
"metrics.yaml": {"data": 1.1},
}
}
}
@pytest.mark.parametrize("use_dvc", [True, False]) @pytest.mark.parametrize("use_dvc", [True, False])
@@ -276,10 +270,7 @@ def test_metrics_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
) )
with (tmp_dir / "dvc.yaml").modify() as d: with (tmp_dir / "dvc.yaml").modify() as d:
# trying to make an output overlaps error # trying to make an output overlaps error
d["stages"]["corrupted-stage"] = { d["stages"]["corrupted-stage"] = {"cmd": "mkdir data", "outs": ["data"]}
"cmd": "mkdir data",
"outs": ["data"],
}
# running by clearing and not clearing stuffs # running by clearing and not clearing stuffs
# so as it works even for optimized cases # so as it works even for optimized cases
+1 -5
View File
@@ -238,11 +238,7 @@ def test_diff_targeted(tmp_dir, scm, dvc, run_copy):
def test_diff_without_targets_specified(tmp_dir, dvc, scm, file): def test_diff_without_targets_specified(tmp_dir, dvc, scm, file):
params_file = tmp_dir / file params_file = tmp_dir / file
params_file.dump({"foo": {"bar": "bar"}, "x": "0"}) params_file.dump({"foo": {"bar": "bar"}, "x": "0"})
dvc.stage.add( dvc.stage.add(name="test", cmd=f"echo {file}", params=[{file: None}])
name="test",
cmd=f"echo {file}",
params=[{file: None}],
)
scm.add_commit([params_file, "dvc.yaml"], message="foo") scm.add_commit([params_file, "dvc.yaml"], message="foo")
params_file.dump({"foo": {"bar": "baz"}, "y": "100"}) params_file.dump({"foo": {"bar": "baz"}, "y": "100"})
+1 -5
View File
@@ -123,11 +123,7 @@ def test_show_without_targets_specified(tmp_dir, dvc, scm, file):
params_file = tmp_dir / file params_file = tmp_dir / file
data = {"foo": {"bar": "bar"}, "x": "0"} data = {"foo": {"bar": "bar"}, "x": "0"}
params_file.dump(data) params_file.dump(data)
dvc.stage.add( dvc.stage.add(name="test", cmd=f"echo {file}", params=[{file: None}])
name="test",
cmd=f"echo {file}",
params=[{file: None}],
)
assert dvc.params.show() == {"": {"data": {file: {"data": data}}}} assert dvc.params.show() == {"": {"data": {file: {"data": data}}}}
+1 -3
View File
@@ -71,9 +71,7 @@ def test_load_vars_with_relpath(tmp_dir, scm, dvc):
revisions = ["HEAD", "workspace"] revisions = ["HEAD", "workspace"]
for rev in dvc.brancher(revs=["HEAD"]): for rev in dvc.brancher(revs=["HEAD"]):
assert rev == revisions.pop() assert rev == revisions.pop()
d = { d = {"vars": [f"../{DEFAULT_PARAMS_FILE}"]}
"vars": [f"../{DEFAULT_PARAMS_FILE}"],
}
resolver = DataResolver(dvc, "subdir", d) resolver = DataResolver(dvc, "subdir", d)
assert resolver.context == deepcopy(DATA) assert resolver.context == deepcopy(DATA)
+2 -10
View File
@@ -155,10 +155,7 @@ def test_plots_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
) )
with (tmp_dir / "dvc.yaml").modify() as d: with (tmp_dir / "dvc.yaml").modify() as d:
# trying to make an output overlaps error # trying to make an output overlaps error
d["stages"]["corrupted-stage"] = { d["stages"]["corrupted-stage"] = {"cmd": "mkdir data", "outs": ["data"]}
"cmd": "mkdir data",
"outs": ["data"],
}
# running by clearing and not clearing stuffs # running by clearing and not clearing stuffs
# so as it works even for optimized cases # so as it works even for optimized cases
@@ -356,12 +353,7 @@ def test_collect_non_existing_dir(tmp_dir, dvc, run_copy_metrics):
), ),
], ],
) )
def test_top_level_plots( def test_top_level_plots(tmp_dir, dvc, plot_config, expected_datafiles):
tmp_dir,
dvc,
plot_config,
expected_datafiles,
):
data = { data = {
"data1.json": [ "data1.json": [
{"a": 1, "b": 0.1, "c": 0.01}, {"a": 1, "b": 0.1, "c": 0.01},
+25 -121
View File
@@ -364,10 +364,7 @@ def test_repro_multiple_params(tmp_dir, dvc):
name="read_params", name="read_params",
deps=["foo"], deps=["foo"],
outs=["bar"], outs=["bar"],
params=[ params=["params2.yaml:lists,floats,name", "answer,floats,nested.nested1"],
"params2.yaml:lists,floats,name",
"answer,floats,nested.nested1",
],
cmd="cat params2.yaml params.yaml > bar", cmd="cat params2.yaml params.yaml > bar",
) )
@@ -528,11 +525,7 @@ def test_repro_skip_pull_if_no_run_cache_is_passed(tmp_dir, dvc, mocker, local_r
assert not spy_pull.called assert not spy_pull.called
def test_repro_no_commit( def test_repro_no_commit(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
@@ -548,16 +541,9 @@ def test_repro_no_commit(
assert not os.path.isdir(dvc.cache.local.path) assert not os.path.isdir(dvc.cache.local.path)
def test_repro_all_pipelines( def test_repro_all_pipelines(mocker, dvc):
mocker,
dvc,
):
stages = [ stages = [
dvc.run( dvc.run(outs=["start.txt"], cmd="echo start > start.txt", name="start"),
outs=["start.txt"],
cmd="echo start > start.txt",
name="start",
),
dvc.run( dvc.run(
deps=["start.txt"], deps=["start.txt"],
outs=["middle.txt"], outs=["middle.txt"],
@@ -588,10 +574,7 @@ def test_repro_all_pipelines(
class TestReproAlreadyCached: class TestReproAlreadyCached:
def test( def test(self, dvc):
self,
dvc,
):
stage = dvc.run( stage = dvc.run(
always_changed=True, always_changed=True,
deps=[], deps=[],
@@ -604,11 +587,7 @@ class TestReproAlreadyCached:
assert run_out.hash_info != repro_out.hash_info assert run_out.hash_info != repro_out.hash_info
def test_force_with_dependencies( def test_force_with_dependencies(self, tmp_dir, dvc):
self,
tmp_dir,
dvc,
):
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
name="datetime", name="datetime",
@@ -654,11 +633,7 @@ def test_repro_shell(tmp_dir, monkeypatch, dvc):
assert (tmp_dir / "shell.txt").read_text().rstrip() == shell assert (tmp_dir / "shell.txt").read_text().rstrip() == shell
def test_cmd_repro( def test_cmd_repro(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
@@ -679,11 +654,7 @@ def test_cmd_repro(
assert ret != 0 assert ret != 0
def test_repro_dep_under_dir( def test_repro_dep_under_dir(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("foo", "foo") tmp_dir.gen("foo", "foo")
tmp_dir.dvc_gen("data", {"file": "file", "sub": {"foo": "foo"}}) tmp_dir.dvc_gen("data", {"file": "file", "sub": {"foo": "foo"}})
@@ -741,11 +712,7 @@ def test_repro_force(tmp_dir, dvc, copy_script):
assert len(stages) == 2 assert len(stages) == 2
def test_repro_changed_code( def test_repro_changed_code(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
@@ -762,11 +729,7 @@ def test_repro_changed_code(
assert len(stages) == 1 assert len(stages) == 1
def test_repro_changed_data( def test_repro_changed_data(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
@@ -783,11 +746,7 @@ def test_repro_changed_data(
assert len(stages) == 2 assert len(stages) == 2
def test_repro_dry( def test_repro_dry(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
@@ -808,11 +767,7 @@ def test_repro_dry(
assert not filecmp.cmp("file1", "bar", shallow=False) assert not filecmp.cmp("file1", "bar", shallow=False)
def test_repro_up_to_date( def test_repro_up_to_date(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
@@ -871,11 +826,7 @@ def test_repro_dry_no_exec(tmp_dir, dvc):
assert ret == 0 assert ret == 0
def test_repro_changed_deep_data( def test_repro_changed_deep_data(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
dvc.run( dvc.run(
@@ -955,11 +906,7 @@ def test_repro_force_downstream_do_not_force_independent_stages(tmp_dir, dvc, ru
assert dvc.reproduce(force_downstream=True) == [foo1, foo2, cat] assert dvc.reproduce(force_downstream=True) == [foo1, foo2, cat]
def test_repro_pipeline( def test_repro_pipeline(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
dvc.run( dvc.run(
@@ -978,11 +925,7 @@ def test_repro_pipeline(
assert len(stages) == 3 assert len(stages) == 3
def test_repro_pipeline_cli( def test_repro_pipeline_cli(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
@@ -995,11 +938,7 @@ def test_repro_pipeline_cli(
assert ret == 0 assert ret == 0
def test_repro_pipelines( def test_repro_pipelines(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
foo_stage, bar_stage = tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) foo_stage, bar_stage = tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
file1_stage = dvc.run( file1_stage = dvc.run(
outs=["file1"], outs=["file1"],
@@ -1021,11 +960,7 @@ def test_repro_pipelines(
} }
def test_repro_pipelines_cli( def test_repro_pipelines_cli(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
dvc.run( dvc.run(
outs=["file1"], outs=["file1"],
@@ -1062,18 +997,11 @@ def test_freeze_non_existing(dvc, target):
assert ret != 0 assert ret != 0
def test_repro_frozen_callback( def test_repro_frozen_callback(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("foo", "foo") tmp_dir.gen("foo", "foo")
# NOTE: purposefully not specifying deps or outs # NOTE: purposefully not specifying deps or outs
# to create a callback stage. # to create a callback stage.
stage = dvc.run( stage = dvc.run(cmd="python copy.py foo file1", name="copy-FOO-file1")
cmd="python copy.py foo file1",
name="copy-FOO-file1",
)
stages = dvc.reproduce(stage.addressing) stages = dvc.reproduce(stage.addressing)
assert len(stages) == 1 assert len(stages) == 1
@@ -1087,11 +1015,7 @@ def test_repro_frozen_callback(
assert len(stages) == 1 assert len(stages) == 1
def test_repro_frozen_unchanged( def test_repro_frozen_unchanged(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
""" """
Check that freezing/unfreezing doesn't affect stage state Check that freezing/unfreezing doesn't affect stage state
""" """
@@ -1155,11 +1079,7 @@ def test_repro_metrics_add_unchanged(tmp_dir, dvc, copy_script):
assert len(stages) == 0 assert len(stages) == 0
def test_repro_phony( def test_repro_phony(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
dvc.run( dvc.run(
@@ -1176,11 +1096,7 @@ def test_repro_phony(
assert filecmp.cmp("file1", "bar", shallow=False) assert filecmp.cmp("file1", "bar", shallow=False)
def test_non_existing_output( def test_non_existing_output(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
@@ -1195,11 +1111,7 @@ def test_non_existing_output(
dvc.reproduce(stage.addressing) dvc.reproduce(stage.addressing)
def test_repro_data_source( def test_repro_data_source(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar") tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
stage = dvc.run( stage = dvc.run(
@@ -1216,11 +1128,7 @@ def test_repro_data_source(
assert stages[0].outs[0].hash_info.value == file_md5("bar") assert stages[0].outs[0].hash_info.value == file_md5("bar")
def test_repro_changed_dir( def test_repro_changed_dir(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen({"foo": "foo", "bar": "bar"}) tmp_dir.gen({"foo": "foo", "bar": "bar"})
shutil.copyfile("foo", "file") shutil.copyfile("foo", "file")
@@ -1241,11 +1149,7 @@ def test_repro_changed_dir(
assert len(stages) == 1 assert len(stages) == 1
def test_repro_changed_dir_data( def test_repro_changed_dir_data(tmp_dir, dvc, copy_script):
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen({"data": {"foo": "foo"}, "bar": "bar"}) tmp_dir.gen({"data": {"foo": "foo"}, "bar": "bar"})
stage = dvc.run( stage = dvc.run(
outs=["dir"], outs=["dir"],
+2 -8
View File
@@ -754,10 +754,7 @@ def test_add_file_in_symlink_dir(make_tmp_dir, tmp_dir, dvc):
def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys): def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys):
tmp_dir.gen("foo", "foo") tmp_dir.gen("foo", "foo")
mocker.patch( mocker.patch("dvc_data.hashfile.checkout.test_links", return_value=[])
"dvc_data.hashfile.checkout.test_links",
return_value=[],
)
dvc.add("foo") dvc.add("foo")
err = capsys.readouterr()[1] err = capsys.readouterr()[1]
assert "reconfigure cache types" in err assert "reconfigure cache types" in err
@@ -981,10 +978,7 @@ def test_add_does_not_remove_stage_file_on_failure(tmp_dir, dvc, mocker, target)
dvcfile_contents = (tmp_dir / stage.path).read_text() dvcfile_contents = (tmp_dir / stage.path).read_text()
exc_msg = f"raising error from mocked '{target}'" exc_msg = f"raising error from mocked '{target}'"
mocker.patch( mocker.patch(target, side_effect=DvcException(exc_msg))
target,
side_effect=DvcException(exc_msg),
)
with pytest.raises(DvcException, match=exc_msg): with pytest.raises(DvcException, match=exc_msg):
dvc.add("foo") dvc.add("foo")
+2 -12
View File
@@ -186,12 +186,7 @@ def test_checkout_empty_dir(tmp_dir, dvc):
def test_checkout_not_cached_file(tmp_dir, dvc): def test_checkout_not_cached_file(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo") tmp_dir.dvc_gen("foo", "foo")
dvc.run( dvc.run(cmd="cp foo bar", deps=["foo"], outs_no_cache=["bar"], name="copy-file")
cmd="cp foo bar",
deps=["foo"],
outs_no_cache=["bar"],
name="copy-file",
)
stats = dvc.checkout(force=True) stats = dvc.checkout(force=True)
assert not any(stats.values()) assert not any(stats.values())
@@ -520,12 +515,7 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link):
def test_checkout_with_deps(tmp_dir, dvc): def test_checkout_with_deps(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo"}) tmp_dir.dvc_gen({"foo": "foo"})
dvc.run( dvc.run(cmd="echo foo > bar", outs=["bar"], deps=["foo"], name="copy-file")
cmd="echo foo > bar",
outs=["bar"],
deps=["foo"],
name="copy-file",
)
(tmp_dir / "bar").unlink() (tmp_dir / "bar").unlink()
(tmp_dir / "foo").unlink() (tmp_dir / "foo").unlink()
+2 -7
View File
@@ -10,9 +10,7 @@ from dvc.cli import main
from dvc.exceptions import CheckoutError from dvc.exceptions import CheckoutError
from dvc.repo.open_repo import clean_repos from dvc.repo.open_repo import clean_repos
from dvc.stage.exceptions import StageNotFound from dvc.stage.exceptions import StageNotFound
from dvc.testing.remote_tests import ( # noqa: F401 from dvc.testing.remote_tests import TestRemote # noqa: F401
TestRemote,
)
from dvc.utils.fs import remove from dvc.utils.fs import remove
from dvc_data.hashfile.db import HashFileDB from dvc_data.hashfile.db import HashFileDB
from dvc_data.hashfile.db.local import LocalHashFileDB from dvc_data.hashfile.db.local import LocalHashFileDB
@@ -177,10 +175,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
assert bar in caplog.text assert bar in caplog.text
caplog.clear() caplog.clear()
assert dvc.status(cloud=True) == { assert dvc.status(cloud=True) == {"bar": "missing", "foo": "missing"}
"bar": "missing",
"foo": "missing",
}
assert header not in caplog.text assert header not in caplog.text
assert foo not in caplog.text assert foo not in caplog.text
assert bar not in caplog.text assert bar not in caplog.text
+6 -23
View File
@@ -84,10 +84,7 @@ def test_directory(M, tmp_dir, dvc, scm):
assert dvc.data_status(granular=True, untracked_files="all") == { assert dvc.data_status(granular=True, untracked_files="all") == {
**EMPTY_STATUS, **EMPTY_STATUS,
"committed": { "committed": {
"added": M.unordered( "added": M.unordered(join("dir", "bar"), join("dir", "foobar")),
join("dir", "bar"),
join("dir", "foobar"),
),
"modified": [join("dir", "")], "modified": [join("dir", "")],
}, },
"uncommitted": { "uncommitted": {
@@ -137,10 +134,7 @@ def test_new_empty_git_repo(M, tmp_dir, scm):
dvc = Repo.init() dvc = Repo.init()
assert dvc.data_status() == { assert dvc.data_status() == {
**EMPTY_STATUS, **EMPTY_STATUS,
"git": M.dict( "git": M.dict(is_empty=True, is_dirty=True),
is_empty=True,
is_dirty=True,
),
} }
@@ -180,10 +174,7 @@ def test_outs_with_no_hashes(M, tmp_dir, dvc, scm):
dvc.stage.add(single_stage=True, outs=["bar"]) dvc.stage.add(single_stage=True, outs=["bar"])
dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar") dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
expected_output = { expected_output = {**EMPTY_STATUS, "git": M.dict()}
**EMPTY_STATUS,
"git": M.dict(),
}
assert dvc.data_status() == expected_output assert dvc.data_status() == expected_output
assert dvc.data_status(granular=True) == expected_output assert dvc.data_status(granular=True) == expected_output
@@ -251,10 +242,7 @@ def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
"untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"), "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
"committed": {"added": M.unordered("foobar", join("dir", ""))}, "committed": {"added": M.unordered("foobar", join("dir", ""))},
"uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))}, "uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))},
"not_in_cache": M.unordered( "not_in_cache": M.unordered("foobar", join("dir", "")),
"foobar",
join("dir", ""),
),
"git": M.dict(), "git": M.dict(),
} }
@@ -297,10 +285,7 @@ def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
} }
assert dvc.data_status(granular=True) == { assert dvc.data_status(granular=True) == {
**EMPTY_STATUS, **EMPTY_STATUS,
"not_in_cache": M.unordered( "not_in_cache": M.unordered("foobar", join("dir", "")),
"foobar",
join("dir", ""),
),
"uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))}, "uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))},
"git": M.dict(), "git": M.dict(),
"unchanged": M.unordered("foobar", join("dir", "")), "unchanged": M.unordered("foobar", join("dir", "")),
@@ -389,9 +374,7 @@ def test_missing_dir_object_from_index(M, tmp_dir, dvc, scm):
} }
assert dvc.data_status(granular=True) == { assert dvc.data_status(granular=True) == {
**EMPTY_STATUS, **EMPTY_STATUS,
"committed": { "committed": {"modified": [join("dir", "")]},
"modified": [join("dir", "")],
},
"uncommitted": {"unknown": [join("dir", "foobar")]}, "uncommitted": {"unknown": [join("dir", "foobar")]},
"not_in_cache": [join("dir", "")], "not_in_cache": [join("dir", "")],
"git": M.dict(), "git": M.dict(),
+1 -4
View File
@@ -106,10 +106,7 @@ def test_load_all_multistage(tmp_dir, dvc):
metrics=["bar2"], metrics=["bar2"],
always_changed=True, always_changed=True,
) )
assert set(load_file(dvc, PROJECT_FILE).stages.values()) == { assert set(load_file(dvc, PROJECT_FILE).stages.values()) == {stage2, stage1}
stage2,
stage1,
}
def test_load_all_singlestage(tmp_dir, dvc): def test_load_all_singlestage(tmp_dir, dvc):
+3 -12
View File
@@ -93,10 +93,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
dest = tmp_dir / "file" dest = tmp_dir / "file"
with external_repo(os.fspath(erepo_dir)) as repo: with external_repo(os.fspath(erepo_dir)) as repo:
repo.dvcfs.get( repo.dvcfs.get("subdir/file", os.fspath(dest))
"subdir/file",
os.fspath(dest),
)
assert dest.is_file() assert dest.is_file()
assert dest.read_text() == "contents" assert dest.read_text() == "contents"
@@ -191,10 +188,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo") subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo: with external_repo(os.fspath(erepo_dir)) as repo:
repo.dvcfs.get( repo.dvcfs.get("dir", os.fspath(tmp_dir / "out"))
"dir",
os.fspath(tmp_dir / "out"),
)
expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"} expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"}
assert (tmp_dir / "out").read_text() == expected_files assert (tmp_dir / "out").read_text() == expected_files
@@ -246,9 +240,6 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo") subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
with external_repo(os.fspath(erepo_dir)) as repo: with external_repo(os.fspath(erepo_dir)) as repo:
repo.dvcfs.get( repo.dvcfs.get("dir", os.fspath(tmp_dir / "out"))
"dir",
os.fspath(tmp_dir / "out"),
)
# subrepo files should not be here # subrepo files should not be here
assert (tmp_dir / "out").read_text() == scm_files assert (tmp_dir / "out").read_text() == scm_files

Some files were not shown because too many files have changed in this diff Show More