mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
nit: fit statement/expr in the same line instead of hanging (#10267)
The code are wrapped due to us using line-length=79 before, but now some of the code can easily fit in 88 chars. The formatter did not change this automatically when we changed line-length, because of magic trailing commas that gets added when they turn into multiple lines. Black/ruff wrapping them in parens and splitting them into multiple lines was making it difficult to read.
This commit is contained in:
@@ -5,10 +5,7 @@ from PyInstaller.utils.hooks import collect_submodules, is_module_or_submodule
|
|||||||
# Celery dynamically imports most celery internals at runtime
|
# Celery dynamically imports most celery internals at runtime
|
||||||
# pyinstaller hook must expose all modules loaded by
|
# pyinstaller hook must expose all modules loaded by
|
||||||
# kombu.utils.imports:symbol_by_name()
|
# kombu.utils.imports:symbol_by_name()
|
||||||
_EXCLUDES = (
|
_EXCLUDES = ("celery.bin", "celery.contrib")
|
||||||
"celery.bin",
|
|
||||||
"celery.contrib",
|
|
||||||
)
|
|
||||||
hiddenimports = collect_submodules(
|
hiddenimports = collect_submodules(
|
||||||
"celery",
|
"celery",
|
||||||
filter=lambda name: not any(
|
filter=lambda name: not any(
|
||||||
|
|||||||
+2
-10
@@ -57,11 +57,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
|||||||
class _OpenContextManager(GCM):
|
class _OpenContextManager(GCM):
|
||||||
def __init__(self, func, args, kwds):
|
def __init__(self, func, args, kwds):
|
||||||
self.gen = func(*args, **kwds)
|
self.gen = func(*args, **kwds)
|
||||||
self.func, self.args, self.kwds = ( # type: ignore[assignment]
|
self.func, self.args, self.kwds = (func, args, kwds) # type: ignore[assignment]
|
||||||
func,
|
|
||||||
args,
|
|
||||||
kwds,
|
|
||||||
)
|
|
||||||
|
|
||||||
def __getattr__(self, name):
|
def __getattr__(self, name):
|
||||||
raise AttributeError("dvc.api.open() should be used in a with statement.")
|
raise AttributeError("dvc.api.open() should be used in a with statement.")
|
||||||
@@ -262,11 +258,7 @@ def _open(
|
|||||||
fs_path = fs.from_os_path(path)
|
fs_path = fs.from_os_path(path)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with fs.open(
|
with fs.open(fs_path, mode=mode, encoding=encoding) as fobj:
|
||||||
fs_path,
|
|
||||||
mode=mode,
|
|
||||||
encoding=encoding,
|
|
||||||
) as fobj:
|
|
||||||
yield fobj
|
yield fobj
|
||||||
except FileNotFoundError as exc:
|
except FileNotFoundError as exc:
|
||||||
raise FileMissingError(path) from exc
|
raise FileMissingError(path) from exc
|
||||||
|
|||||||
+2
-8
@@ -144,15 +144,9 @@ def migrate_2_to_3(repo: "Repo", dry: bool = False):
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
with TqdmCallback(
|
with TqdmCallback(desc="Computing DVC 3.0 hashes", unit="files") as cb:
|
||||||
desc="Computing DVC 3.0 hashes",
|
|
||||||
unit="files",
|
|
||||||
) as cb:
|
|
||||||
migration = prepare(src, dest, callback=cb)
|
migration = prepare(src, dest, callback=cb)
|
||||||
|
|
||||||
with TqdmCallback(
|
with TqdmCallback(desc="Migrating to DVC 3.0 cache", unit="files") as cb:
|
||||||
desc="Migrating to DVC 3.0 cache",
|
|
||||||
unit="files",
|
|
||||||
) as cb:
|
|
||||||
count = migrate(migration, callback=cb)
|
count = migrate(migration, callback=cb)
|
||||||
ui.write(f"Migrated {count} files to DVC 3.0 cache location.")
|
ui.write(f"Migrated {count} files to DVC 3.0 cache location.")
|
||||||
|
|||||||
@@ -107,8 +107,4 @@ def _add_run_common(parser):
|
|||||||
default=None,
|
default=None,
|
||||||
help="Custom commit message to use when committing the experiment.",
|
help="Custom commit message to use when committing the experiment.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
|
||||||
"-M", # obsolete
|
|
||||||
dest="message",
|
|
||||||
help=argparse.SUPPRESS,
|
|
||||||
)
|
|
||||||
|
|||||||
@@ -93,9 +93,5 @@ def add_parser(experiments_subparsers, parent_parser):
|
|||||||
default=None,
|
default=None,
|
||||||
help="Custom commit message to use when committing the experiment.",
|
help="Custom commit message to use when committing the experiment.",
|
||||||
)
|
)
|
||||||
save_parser.add_argument(
|
save_parser.add_argument("-M", dest="message", help=argparse.SUPPRESS) # obsolete
|
||||||
"-M", # obsolete
|
|
||||||
dest="message",
|
|
||||||
help=argparse.SUPPRESS,
|
|
||||||
)
|
|
||||||
save_parser.set_defaults(func=CmdExperimentsSave)
|
save_parser.set_defaults(func=CmdExperimentsSave)
|
||||||
|
|||||||
@@ -119,11 +119,7 @@ def add_parser(subparsers, parent_parser):
|
|||||||
"specified by '--remote') in the target repository."
|
"specified by '--remote') in the target repository."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
list_parser.add_argument(
|
list_parser.add_argument("--size", action="store_true", help="Show sizes.")
|
||||||
"--size",
|
|
||||||
action="store_true",
|
|
||||||
help="Show sizes.",
|
|
||||||
)
|
|
||||||
list_parser.add_argument(
|
list_parser.add_argument(
|
||||||
"path",
|
"path",
|
||||||
nargs="?",
|
nargs="?",
|
||||||
|
|||||||
@@ -38,16 +38,9 @@ def add_parser(subparsers, parent_parser):
|
|||||||
"url", help="See `dvc import-url -h` for full list of supported URLs."
|
"url", help="See `dvc import-url -h` for full list of supported URLs."
|
||||||
)
|
)
|
||||||
lsurl_parser.add_argument(
|
lsurl_parser.add_argument(
|
||||||
"-R",
|
"-R", "--recursive", action="store_true", help="Recursively list files."
|
||||||
"--recursive",
|
|
||||||
action="store_true",
|
|
||||||
help="Recursively list files.",
|
|
||||||
)
|
|
||||||
lsurl_parser.add_argument(
|
|
||||||
"--size",
|
|
||||||
action="store_true",
|
|
||||||
help="Show sizes.",
|
|
||||||
)
|
)
|
||||||
|
lsurl_parser.add_argument("--size", action="store_true", help="Show sizes.")
|
||||||
lsurl_parser.add_argument(
|
lsurl_parser.add_argument(
|
||||||
"--fs-config",
|
"--fs-config",
|
||||||
type=str,
|
type=str,
|
||||||
|
|||||||
+3
-15
@@ -34,21 +34,12 @@ def _show_json(
|
|||||||
name = renderer.name
|
name = renderer.name
|
||||||
data[name] = to_json(renderer, split)
|
data[name] = to_json(renderer, split)
|
||||||
all_errors.extend(
|
all_errors.extend(
|
||||||
{
|
{"name": name, "rev": rev, "source": source, **encode_exception(e)}
|
||||||
"name": name,
|
|
||||||
"rev": rev,
|
|
||||||
"source": source,
|
|
||||||
**encode_exception(e),
|
|
||||||
}
|
|
||||||
for rev, per_rev_src_errors in src_errors.items()
|
for rev, per_rev_src_errors in src_errors.items()
|
||||||
for source, e in per_rev_src_errors.items()
|
for source, e in per_rev_src_errors.items()
|
||||||
)
|
)
|
||||||
all_errors.extend(
|
all_errors.extend(
|
||||||
{
|
{"name": name, "rev": rev, **encode_exception(e)}
|
||||||
"name": name,
|
|
||||||
"rev": rev,
|
|
||||||
**encode_exception(e),
|
|
||||||
}
|
|
||||||
for rev, e in def_errors.items()
|
for rev, e in def_errors.items()
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -101,10 +92,7 @@ class CmdPlots(CmdBase):
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
try:
|
try:
|
||||||
plots_data = self._func(
|
plots_data = self._func(targets=self.args.targets, props=self._props())
|
||||||
targets=self.args.targets,
|
|
||||||
props=self._props(),
|
|
||||||
)
|
|
||||||
|
|
||||||
if not plots_data and not self.args.json:
|
if not plots_data and not self.args.json:
|
||||||
ui.error_write(
|
ui.error_write(
|
||||||
|
|||||||
@@ -2,14 +2,7 @@ from dvc.cli import formatter
|
|||||||
from dvc.cli.utils import append_doc_link
|
from dvc.cli.utils import append_doc_link
|
||||||
from dvc.commands.queue import kill, logs, remove, start, status, stop
|
from dvc.commands.queue import kill, logs, remove, start, status, stop
|
||||||
|
|
||||||
SUB_COMMANDS = [
|
SUB_COMMANDS = [start, stop, status, logs, remove, kill]
|
||||||
start,
|
|
||||||
stop,
|
|
||||||
status,
|
|
||||||
logs,
|
|
||||||
remove,
|
|
||||||
kill,
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def add_parser(subparsers, parent_parser):
|
def add_parser(subparsers, parent_parser):
|
||||||
|
|||||||
@@ -45,9 +45,5 @@ def add_parser(queue_subparsers, parent_parser):
|
|||||||
),
|
),
|
||||||
action="store_true",
|
action="store_true",
|
||||||
)
|
)
|
||||||
queue_logs_parser.add_argument(
|
queue_logs_parser.add_argument("task", help="Task to show.", metavar="<task>")
|
||||||
"task",
|
|
||||||
help="Task to show.",
|
|
||||||
metavar="<task>",
|
|
||||||
)
|
|
||||||
queue_logs_parser.set_defaults(func=CmdQueueLogs)
|
queue_logs_parser.set_defaults(func=CmdQueueLogs)
|
||||||
|
|||||||
+1
-7
@@ -9,13 +9,7 @@ from collections.abc import (
|
|||||||
)
|
)
|
||||||
from itertools import chain, repeat, zip_longest
|
from itertools import chain, repeat, zip_longest
|
||||||
from operator import itemgetter
|
from operator import itemgetter
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Optional, Union, overload
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
overload,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import reraise
|
from funcy import reraise
|
||||||
|
|
||||||
|
|||||||
+1
-6
@@ -107,12 +107,7 @@ def _posix_detached_subprocess(args: Sequence[str], **kwargs) -> int:
|
|||||||
os.close(read_end)
|
os.close(read_end)
|
||||||
return int(pid_str)
|
return int(pid_str)
|
||||||
|
|
||||||
proc = subprocess.Popen(
|
proc = subprocess.Popen(args, shell=False, close_fds=True, **kwargs) # noqa: S603
|
||||||
args,
|
|
||||||
shell=False, # noqa: S603
|
|
||||||
close_fds=True,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
os.close(read_end)
|
os.close(read_end)
|
||||||
os.write(write_end, str(proc.pid).encode("utf8"))
|
os.write(write_end, str(proc.pid).encode("utf8"))
|
||||||
os.close(write_end)
|
os.close(write_end)
|
||||||
|
|||||||
@@ -59,15 +59,7 @@ def loads_from(stage, s_list, erepo=None, fs_config=None, db=None):
|
|||||||
info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
|
info = {RepoDependency.PARAM_REPO: erepo} if erepo else {}
|
||||||
if db:
|
if db:
|
||||||
info.update({"db": db})
|
info.update({"db": db})
|
||||||
return [
|
return [_get(stage, s, info.copy(), fs_config=fs_config) for s in s_list]
|
||||||
_get(
|
|
||||||
stage,
|
|
||||||
s,
|
|
||||||
info.copy(),
|
|
||||||
fs_config=fs_config,
|
|
||||||
)
|
|
||||||
for s in s_list
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _merge_params(s_list) -> dict[str, list[str]]:
|
def _merge_params(s_list) -> dict[str, list[str]]:
|
||||||
|
|||||||
@@ -76,10 +76,7 @@ class ParamsDependency(Dependency):
|
|||||||
self.params = list(params) if params else []
|
self.params = list(params) if params else []
|
||||||
hash_info = HashInfo()
|
hash_info = HashInfo()
|
||||||
if isinstance(params, dict):
|
if isinstance(params, dict):
|
||||||
hash_info = HashInfo(
|
hash_info = HashInfo(self.PARAM_PARAMS, params) # type: ignore[arg-type]
|
||||||
self.PARAM_PARAMS,
|
|
||||||
params, # type: ignore[arg-type]
|
|
||||||
)
|
|
||||||
repo = repo or stage.repo
|
repo = repo or stage.repo
|
||||||
path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE)
|
path = path or os.path.join(repo.root_dir, self.DEFAULT_PARAMS_FILE)
|
||||||
super().__init__(stage, path, repo=repo)
|
super().__init__(stage, path, repo=repo)
|
||||||
@@ -103,10 +100,7 @@ class ParamsDependency(Dependency):
|
|||||||
for param in self.params:
|
for param in self.params:
|
||||||
if param in values:
|
if param in values:
|
||||||
info[param] = values[param]
|
info[param] = values[param]
|
||||||
self.hash_info = HashInfo(
|
self.hash_info = HashInfo(self.PARAM_PARAMS, info) # type: ignore[arg-type]
|
||||||
self.PARAM_PARAMS,
|
|
||||||
info, # type: ignore[arg-type]
|
|
||||||
)
|
|
||||||
|
|
||||||
def read_params(
|
def read_params(
|
||||||
self, flatten: bool = True, **kwargs: typing.Any
|
self, flatten: bool = True, **kwargs: typing.Any
|
||||||
|
|||||||
+1
-9
@@ -1,14 +1,6 @@
|
|||||||
import contextlib
|
import contextlib
|
||||||
import os
|
import os
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, TypeVar, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Callable,
|
|
||||||
ClassVar,
|
|
||||||
Optional,
|
|
||||||
TypeVar,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from dvc.exceptions import DvcException
|
from dvc.exceptions import DvcException
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
|
|||||||
+2
-12
@@ -52,10 +52,7 @@ def download(
|
|||||||
|
|
||||||
from .callbacks import TqdmCallback
|
from .callbacks import TqdmCallback
|
||||||
|
|
||||||
with TqdmCallback(
|
with TqdmCallback(desc=f"Downloading {fs.name(fs_path)}", unit="files") as cb:
|
||||||
desc=f"Downloading {fs.name(fs_path)}",
|
|
||||||
unit="files",
|
|
||||||
) as cb:
|
|
||||||
# NOTE: We use dvc-objects generic.copy over fs.get since it makes file
|
# NOTE: We use dvc-objects generic.copy over fs.get since it makes file
|
||||||
# download atomic and avoids fsspec glob/regex path expansion.
|
# download atomic and avoids fsspec glob/regex path expansion.
|
||||||
if fs.isdir(fs_path):
|
if fs.isdir(fs_path):
|
||||||
@@ -76,14 +73,7 @@ def download(
|
|||||||
lfs_prefetch(fs, from_infos)
|
lfs_prefetch(fs, from_infos)
|
||||||
cb.set_size(len(from_infos))
|
cb.set_size(len(from_infos))
|
||||||
jobs = jobs or fs.jobs
|
jobs = jobs or fs.jobs
|
||||||
generic.copy(
|
generic.copy(fs, from_infos, localfs, to_infos, callback=cb, batch_size=jobs)
|
||||||
fs,
|
|
||||||
from_infos,
|
|
||||||
localfs,
|
|
||||||
to_infos,
|
|
||||||
callback=cb,
|
|
||||||
batch_size=jobs,
|
|
||||||
)
|
|
||||||
return len(to_infos)
|
return len(to_infos)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+1
-3
@@ -22,9 +22,7 @@ class DataFileSystem(FileSystem):
|
|||||||
return config
|
return config
|
||||||
|
|
||||||
@functools.cached_property
|
@functools.cached_property
|
||||||
def fs(
|
def fs(self) -> "_DataFileSystem":
|
||||||
self,
|
|
||||||
) -> "_DataFileSystem":
|
|
||||||
from dvc_data.fs import DataFileSystem as _DataFileSystem
|
from dvc_data.fs import DataFileSystem as _DataFileSystem
|
||||||
|
|
||||||
return _DataFileSystem(**self.fs_args)
|
return _DataFileSystem(**self.fs_args)
|
||||||
|
|||||||
+1
-3
@@ -39,9 +39,7 @@ class GitFileSystem(FileSystem):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@functools.cached_property
|
@functools.cached_property
|
||||||
def fs(
|
def fs(self) -> "FsspecGitFileSystem":
|
||||||
self,
|
|
||||||
) -> "FsspecGitFileSystem":
|
|
||||||
from scmrepo.fs import GitFileSystem as FsspecGitFileSystem
|
from scmrepo.fs import GitFileSystem as FsspecGitFileSystem
|
||||||
|
|
||||||
return FsspecGitFileSystem(**self.fs_args)
|
return FsspecGitFileSystem(**self.fs_args)
|
||||||
|
|||||||
+1
-3
@@ -185,9 +185,7 @@ class HardlinkLock(flufl.lock.Lock, LockBase):
|
|||||||
|
|
||||||
if self._tmp_dir is not None:
|
if self._tmp_dir is not None:
|
||||||
# Under Windows file path length is limited so we hash it
|
# Under Windows file path length is limited so we hash it
|
||||||
hasher = hashlib.md5( # noqa: S324
|
hasher = hashlib.md5(self._claimfile.encode()) # noqa: S324
|
||||||
self._claimfile.encode()
|
|
||||||
)
|
|
||||||
filename = hasher.hexdigest()
|
filename = hasher.hexdigest()
|
||||||
self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
|
self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
|
||||||
|
|
||||||
|
|||||||
+9
-43
@@ -167,11 +167,7 @@ def _merge_data(s_list):
|
|||||||
|
|
||||||
@collecting
|
@collecting
|
||||||
def load_from_pipeline(stage, data, typ="outs"):
|
def load_from_pipeline(stage, data, typ="outs"):
|
||||||
if typ not in (
|
if typ not in (stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS):
|
||||||
stage.PARAM_OUTS,
|
|
||||||
stage.PARAM_METRICS,
|
|
||||||
stage.PARAM_PLOTS,
|
|
||||||
):
|
|
||||||
raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
|
raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
|
||||||
|
|
||||||
metric = typ == stage.PARAM_METRICS
|
metric = typ == stage.PARAM_METRICS
|
||||||
@@ -197,24 +193,13 @@ def load_from_pipeline(stage, data, typ="outs"):
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
yield _get(
|
yield _get(stage, path, info={}, plot=plt_d or plot, metric=metric, **extra)
|
||||||
stage,
|
|
||||||
path,
|
|
||||||
info={},
|
|
||||||
plot=plt_d or plot,
|
|
||||||
metric=metric,
|
|
||||||
**extra,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def split_file_meta_from_cloud(entry: dict) -> dict:
|
def split_file_meta_from_cloud(entry: dict) -> dict:
|
||||||
if remote_name := entry.pop(Meta.PARAM_REMOTE, None):
|
if remote_name := entry.pop(Meta.PARAM_REMOTE, None):
|
||||||
remote_meta = {}
|
remote_meta = {}
|
||||||
for key in (
|
for key in (S3_PARAM_CHECKSUM, HDFS_PARAM_CHECKSUM, Meta.PARAM_VERSION_ID):
|
||||||
S3_PARAM_CHECKSUM,
|
|
||||||
HDFS_PARAM_CHECKSUM,
|
|
||||||
Meta.PARAM_VERSION_ID,
|
|
||||||
):
|
|
||||||
if value := entry.pop(key, None):
|
if value := entry.pop(key, None):
|
||||||
remote_meta[key] = value
|
remote_meta[key] = value
|
||||||
|
|
||||||
@@ -433,10 +418,7 @@ class Output:
|
|||||||
hash_name = meta_name = self.fs.PARAM_CHECKSUM
|
hash_name = meta_name = self.fs.PARAM_CHECKSUM
|
||||||
assert hash_name
|
assert hash_name
|
||||||
|
|
||||||
hash_info = HashInfo(
|
hash_info = HashInfo(name=hash_name, value=getattr(self.meta, meta_name, None))
|
||||||
name=hash_name,
|
|
||||||
value=getattr(self.meta, meta_name, None),
|
|
||||||
)
|
|
||||||
return hash_name, hash_info
|
return hash_name, hash_info
|
||||||
|
|
||||||
def _compute_meta_hash_info_from_files(self) -> None:
|
def _compute_meta_hash_info_from_files(self) -> None:
|
||||||
@@ -515,10 +497,7 @@ class Output:
|
|||||||
if self.fs.isabs(self.def_path):
|
if self.fs.isabs(self.def_path):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
return self.repo and self.fs.isin(
|
return self.repo and self.fs.isin(self.fs_path, self.repo.root_dir)
|
||||||
self.fs_path,
|
|
||||||
self.repo.root_dir,
|
|
||||||
)
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def use_scm_ignore(self):
|
def use_scm_ignore(self):
|
||||||
@@ -813,20 +792,13 @@ class Output:
|
|||||||
def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
|
def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
|
||||||
prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
|
prefix = self.fs.parts(self.fs.relpath(filter_info, self.fs_path))
|
||||||
staging, _, obj = self._build(
|
staging, _, obj = self._build(
|
||||||
self.cache,
|
self.cache, self.fs_path, self.fs, self.hash_name, ignore=self.dvcignore
|
||||||
self.fs_path,
|
|
||||||
self.fs,
|
|
||||||
self.hash_name,
|
|
||||||
ignore=self.dvcignore,
|
|
||||||
)
|
)
|
||||||
assert isinstance(obj, Tree)
|
assert isinstance(obj, Tree)
|
||||||
save_obj = obj.filter(prefix)
|
save_obj = obj.filter(prefix)
|
||||||
assert isinstance(save_obj, Tree)
|
assert isinstance(save_obj, Tree)
|
||||||
checkout_obj = save_obj.get_obj(self.cache, prefix)
|
checkout_obj = save_obj.get_obj(self.cache, prefix)
|
||||||
with TqdmCallback(
|
with TqdmCallback(desc=f"Committing {self} to cache", unit="file") as cb:
|
||||||
desc=f"Committing {self} to cache",
|
|
||||||
unit="file",
|
|
||||||
) as cb:
|
|
||||||
otransfer(
|
otransfer(
|
||||||
staging,
|
staging,
|
||||||
self.cache,
|
self.cache,
|
||||||
@@ -1286,10 +1258,7 @@ class Output:
|
|||||||
|
|
||||||
self.hash_info = merged.hash_info
|
self.hash_info = merged.hash_info
|
||||||
self.files = None
|
self.files = None
|
||||||
self.meta = Meta(
|
self.meta = Meta(size=du(self.cache, merged), nfiles=len(merged))
|
||||||
size=du(self.cache, merged),
|
|
||||||
nfiles=len(merged),
|
|
||||||
)
|
|
||||||
|
|
||||||
def unstage(self, path: str) -> tuple["Meta", "Tree"]:
|
def unstage(self, path: str) -> tuple["Meta", "Tree"]:
|
||||||
from pygtrie import Trie
|
from pygtrie import Trie
|
||||||
@@ -1429,10 +1398,7 @@ class Output:
|
|||||||
|
|
||||||
assert staging
|
assert staging
|
||||||
assert obj.hash_info
|
assert obj.hash_info
|
||||||
with TqdmCallback(
|
with TqdmCallback(desc=f"Adding {self} to cache", unit="file") as cb:
|
||||||
desc=f"Adding {self} to cache",
|
|
||||||
unit="file",
|
|
||||||
) as cb:
|
|
||||||
otransfer(
|
otransfer(
|
||||||
staging,
|
staging,
|
||||||
self.cache,
|
self.cache,
|
||||||
|
|||||||
@@ -3,13 +3,7 @@ import os
|
|||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Mapping, Sequence
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from itertools import product
|
from itertools import product
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
NamedTuple,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import collecting, first, isa, join, reraise
|
from funcy import collecting, first, isa, join, reraise
|
||||||
|
|
||||||
|
|||||||
@@ -28,9 +28,7 @@ class ImageConverter(Converter):
|
|||||||
return img_path
|
return img_path
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _encode_image(
|
def _encode_image(image_data: bytes) -> str:
|
||||||
image_data: bytes,
|
|
||||||
) -> str:
|
|
||||||
base64_str = base64.b64encode(image_data).decode()
|
base64_str = base64.b64encode(image_data).decode()
|
||||||
return f"data:image;base64,{base64_str}"
|
return f"data:image;base64,{base64_str}"
|
||||||
|
|
||||||
@@ -57,10 +55,6 @@ class ImageConverter(Converter):
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
src = self._encode_image(image_data)
|
src = self._encode_image(image_data)
|
||||||
datapoint = {
|
datapoint = {REVISION: revision, FILENAME: filename, SRC: src}
|
||||||
REVISION: revision,
|
|
||||||
FILENAME: filename,
|
|
||||||
SRC: src,
|
|
||||||
}
|
|
||||||
datapoints.append(datapoint)
|
datapoints.append(datapoint)
|
||||||
return datapoints, properties
|
return datapoints, properties
|
||||||
|
|||||||
@@ -41,11 +41,7 @@ def _file_field(*args):
|
|||||||
yield file, field
|
yield file, field
|
||||||
|
|
||||||
|
|
||||||
def _find(
|
def _find(filename: str, field: str, data_series: list[tuple[str, str, Any]]):
|
||||||
filename: str,
|
|
||||||
field: str,
|
|
||||||
data_series: list[tuple[str, str, Any]],
|
|
||||||
):
|
|
||||||
for data_file, data_field, data in data_series:
|
for data_file, data_field, data in data_series:
|
||||||
if data_file == filename and data_field == field:
|
if data_file == filename and data_field == field:
|
||||||
return data_file, data_field, data
|
return data_file, data_field, data
|
||||||
@@ -202,10 +198,7 @@ class VegaConverter(Converter):
|
|||||||
|
|
||||||
# assign "step" if no x provided
|
# assign "step" if no x provided
|
||||||
if not xs:
|
if not xs:
|
||||||
x_file, x_field = (
|
x_file, x_field = None, INDEX
|
||||||
None,
|
|
||||||
INDEX,
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
x_file, x_field = xs[0]
|
x_file, x_field = xs[0]
|
||||||
|
|
||||||
@@ -244,10 +237,7 @@ class VegaConverter(Converter):
|
|||||||
common_prefix_len = 0
|
common_prefix_len = 0
|
||||||
|
|
||||||
props_update["anchors_y_definitions"] = [
|
props_update["anchors_y_definitions"] = [
|
||||||
{
|
{FILENAME: _get_short_y_file(y_file, common_prefix_len), FIELD: y_field}
|
||||||
FILENAME: _get_short_y_file(y_file, common_prefix_len),
|
|
||||||
FIELD: y_field,
|
|
||||||
}
|
|
||||||
for y_file, y_field in ys
|
for y_file, y_field in ys
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -299,9 +289,7 @@ class VegaConverter(Converter):
|
|||||||
|
|
||||||
return all_datapoints, properties
|
return all_datapoints, properties
|
||||||
|
|
||||||
def convert(
|
def convert(self):
|
||||||
self,
|
|
||||||
):
|
|
||||||
"""
|
"""
|
||||||
Convert the data. Fill necessary fields ('x', 'y') and return both
|
Convert the data. Fill necessary fields ('x', 'y') and return both
|
||||||
generated datapoints and updated properties. `x`, `y` values and labels
|
generated datapoints and updated properties. `x`, `y` values and labels
|
||||||
|
|||||||
+5
-28
@@ -3,12 +3,7 @@ from collections import defaultdict
|
|||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from contextlib import AbstractContextManager, contextmanager
|
from contextlib import AbstractContextManager, contextmanager
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Callable, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Callable,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from dvc.exceptions import (
|
from dvc.exceptions import (
|
||||||
DvcException,
|
DvcException,
|
||||||
@@ -179,14 +174,8 @@ class Repo:
|
|||||||
|
|
||||||
self.root_dir: str
|
self.root_dir: str
|
||||||
self.dvc_dir: Optional[str]
|
self.dvc_dir: Optional[str]
|
||||||
(
|
(self.root_dir, self.dvc_dir) = self._get_repo_dirs(
|
||||||
self.root_dir,
|
root_dir=root_dir, fs=self.fs, uninitialized=uninitialized, scm=scm
|
||||||
self.dvc_dir,
|
|
||||||
) = self._get_repo_dirs(
|
|
||||||
root_dir=root_dir,
|
|
||||||
fs=self.fs,
|
|
||||||
uninitialized=uninitialized,
|
|
||||||
scm=scm,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
self._uninitialized = uninitialized
|
self._uninitialized = uninitialized
|
||||||
@@ -271,11 +260,7 @@ class Repo:
|
|||||||
# subrepo
|
# subrepo
|
||||||
relparts = self.fs.relparts(self.root_dir, "/")
|
relparts = self.fs.relparts(self.root_dir, "/")
|
||||||
|
|
||||||
dvc_dir = os.path.join(
|
dvc_dir = os.path.join(self.scm.root_dir, *relparts, self.DVC_DIR)
|
||||||
self.scm.root_dir,
|
|
||||||
*relparts,
|
|
||||||
self.DVC_DIR,
|
|
||||||
)
|
|
||||||
if os.path.exists(dvc_dir):
|
if os.path.exists(dvc_dir):
|
||||||
return dvc_dir
|
return dvc_dir
|
||||||
|
|
||||||
@@ -651,15 +636,7 @@ class Repo:
|
|||||||
btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None)
|
btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None)
|
||||||
|
|
||||||
md5 = hashlib.md5( # noqa: S324
|
md5 = hashlib.md5( # noqa: S324
|
||||||
str(
|
str((root_dir, btime, getpass.getuser(), version_tuple[0], salt)).encode()
|
||||||
(
|
|
||||||
root_dir,
|
|
||||||
btime,
|
|
||||||
getpass.getuser(),
|
|
||||||
version_tuple[0],
|
|
||||||
salt,
|
|
||||||
)
|
|
||||||
).encode()
|
|
||||||
)
|
)
|
||||||
repo_token = md5.hexdigest()
|
repo_token = md5.hexdigest()
|
||||||
return os.path.join(repos_dir, repo_token)
|
return os.path.join(repos_dir, repo_token)
|
||||||
|
|||||||
+1
-6
@@ -1,12 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, NamedTuple, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
NamedTuple,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from dvc.exceptions import (
|
from dvc.exceptions import (
|
||||||
CacheLinkError,
|
CacheLinkError,
|
||||||
|
|||||||
+2
-10
@@ -98,10 +98,7 @@ class Artifacts:
|
|||||||
def read(self) -> dict[str, dict[str, Artifact]]:
|
def read(self) -> dict[str, dict[str, Artifact]]:
|
||||||
"""Read artifacts from dvc.yaml."""
|
"""Read artifacts from dvc.yaml."""
|
||||||
artifacts: dict[str, dict[str, Artifact]] = {}
|
artifacts: dict[str, dict[str, Artifact]] = {}
|
||||||
for (
|
for dvcfile, dvcfile_artifacts in self.repo.index._artifacts.items():
|
||||||
dvcfile,
|
|
||||||
dvcfile_artifacts,
|
|
||||||
) in self.repo.index._artifacts.items():
|
|
||||||
dvcyaml = relpath(dvcfile, self.repo.root_dir)
|
dvcyaml = relpath(dvcfile, self.repo.root_dir)
|
||||||
artifacts[dvcyaml] = {}
|
artifacts[dvcyaml] = {}
|
||||||
for name, value in dvcfile_artifacts.items():
|
for name, value in dvcfile_artifacts.items():
|
||||||
@@ -188,12 +185,7 @@ class Artifacts:
|
|||||||
out = resolve_output(path, out, force=force)
|
out = resolve_output(path, out, force=force)
|
||||||
fs = self.repo.dvcfs
|
fs = self.repo.dvcfs
|
||||||
fs_path = fs.from_os_path(path)
|
fs_path = fs.from_os_path(path)
|
||||||
count = fs_download(
|
count = fs_download(fs, fs_path, os.path.abspath(out), jobs=jobs)
|
||||||
fs,
|
|
||||||
fs_path,
|
|
||||||
os.path.abspath(out),
|
|
||||||
jobs=jobs,
|
|
||||||
)
|
|
||||||
return count, out
|
return count, out
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
+1
-5
@@ -5,11 +5,7 @@ def check_missing(repo, rev=None, max_size=None, types=None):
|
|||||||
from dvc_data.index import StorageKeyError
|
from dvc_data.index import StorageKeyError
|
||||||
|
|
||||||
with repo.switch(rev or "workspace"):
|
with repo.switch(rev or "workspace"):
|
||||||
idx = repo.index.targets_view(
|
idx = repo.index.targets_view(None, max_size=max_size, types=types)
|
||||||
None,
|
|
||||||
max_size=max_size,
|
|
||||||
types=types,
|
|
||||||
)
|
|
||||||
|
|
||||||
index = idx.data["repo"]
|
index = idx.data["repo"]
|
||||||
|
|
||||||
|
|||||||
+6
-29
@@ -122,42 +122,23 @@ def checkout( # noqa: C901
|
|||||||
def onerror(target, exc):
|
def onerror(target, exc):
|
||||||
if target and isinstance(
|
if target and isinstance(
|
||||||
exc,
|
exc,
|
||||||
(
|
(StageFileDoesNotExistError, StageFileBadNameError, NoOutputOrStageError),
|
||||||
StageFileDoesNotExistError,
|
|
||||||
StageFileBadNameError,
|
|
||||||
NoOutputOrStageError,
|
|
||||||
),
|
|
||||||
):
|
):
|
||||||
raise CheckoutErrorSuggestGit(target) from exc
|
raise CheckoutErrorSuggestGit(target) from exc
|
||||||
raise
|
raise
|
||||||
|
|
||||||
view = self.index.targets_view(
|
view = self.index.targets_view(
|
||||||
targets,
|
targets, recursive=recursive, with_deps=with_deps, onerror=onerror
|
||||||
recursive=recursive,
|
|
||||||
with_deps=with_deps,
|
|
||||||
onerror=onerror,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(unit="entry", desc="Building workspace index", leave=True) as pb:
|
||||||
unit="entry",
|
|
||||||
desc="Building workspace index",
|
|
||||||
leave=True,
|
|
||||||
) as pb:
|
|
||||||
old = build_data_index(
|
old = build_data_index(
|
||||||
view,
|
view, self.root_dir, self.fs, compute_hash=True, callback=pb.as_callback()
|
||||||
self.root_dir,
|
|
||||||
self.fs,
|
|
||||||
compute_hash=True,
|
|
||||||
callback=pb.as_callback(),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
new = view.data["repo"]
|
new = view.data["repo"]
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(desc="Comparing indexes", unit="entry", leave=True) as pb:
|
||||||
desc="Comparing indexes",
|
|
||||||
unit="entry",
|
|
||||||
leave=True,
|
|
||||||
) as pb:
|
|
||||||
diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback())
|
diff = compare(old, new, relink=relink, delete=True, callback=pb.as_callback())
|
||||||
|
|
||||||
if not force:
|
if not force:
|
||||||
@@ -175,11 +156,7 @@ def checkout( # noqa: C901
|
|||||||
if self.fs.isin_or_eq(dest_path, out_path):
|
if self.fs.isin_or_eq(dest_path, out_path):
|
||||||
failed.add(out_path)
|
failed.add(out_path)
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(unit="file", desc="Applying changes", leave=True) as pb:
|
||||||
unit="file",
|
|
||||||
desc="Applying changes",
|
|
||||||
leave=True,
|
|
||||||
) as pb:
|
|
||||||
apply(
|
apply(
|
||||||
diff,
|
diff,
|
||||||
self.root_dir,
|
self.root_dir,
|
||||||
|
|||||||
+4
-17
@@ -154,10 +154,7 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
|
|||||||
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
|
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
|
||||||
from .index import build_data_index
|
from .index import build_data_index
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(desc="Building workspace index", unit="entry") as pb:
|
||||||
desc="Building workspace index",
|
|
||||||
unit="entry",
|
|
||||||
) as pb:
|
|
||||||
workspace = build_data_index(
|
workspace = build_data_index(
|
||||||
repo.index,
|
repo.index,
|
||||||
repo.root_dir,
|
repo.root_dir,
|
||||||
@@ -187,10 +184,7 @@ def _diff_head_to_index(
|
|||||||
with repo.switch(head):
|
with repo.switch(head):
|
||||||
head_index = repo.index.data["repo"]
|
head_index = repo.index.data["repo"]
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(desc="Calculating diff between head/index", unit="entry") as pb:
|
||||||
desc="Calculating diff between head/index",
|
|
||||||
unit="entry",
|
|
||||||
) as pb:
|
|
||||||
return _diff(head_index, index, callback=pb.as_callback(), **kwargs)
|
return _diff(head_index, index, callback=pb.as_callback(), **kwargs)
|
||||||
|
|
||||||
|
|
||||||
@@ -223,18 +217,11 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]
|
|||||||
return [repo.fs.relpath(file, start) for file in files]
|
return [repo.fs.relpath(file, start) for file in files]
|
||||||
|
|
||||||
|
|
||||||
def status(
|
def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
|
||||||
repo: "Repo",
|
|
||||||
untracked_files: str = "no",
|
|
||||||
**kwargs: Any,
|
|
||||||
) -> Status:
|
|
||||||
from dvc.scm import NoSCMError, SCMError
|
from dvc.scm import NoSCMError, SCMError
|
||||||
|
|
||||||
head = kwargs.pop("head", "HEAD")
|
head = kwargs.pop("head", "HEAD")
|
||||||
uncommitted_diff = _diff_index_to_wtree(
|
uncommitted_diff = _diff_index_to_wtree(repo, **kwargs)
|
||||||
repo,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
unchanged = set(uncommitted_diff.pop("unchanged", []))
|
unchanged = set(uncommitted_diff.pop("unchanged", []))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
+7
-33
@@ -52,36 +52,22 @@ def _diff(old, new, data_keys, with_missing=False):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
if change.typ == ADD:
|
if change.typ == ADD:
|
||||||
ret["added"].append(
|
ret["added"].append({"path": _path(change.new), "hash": _hash(change.new)})
|
||||||
{
|
|
||||||
"path": _path(change.new),
|
|
||||||
"hash": _hash(change.new),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
elif change.typ == DELETE:
|
elif change.typ == DELETE:
|
||||||
ret["deleted"].append(
|
ret["deleted"].append(
|
||||||
{
|
{"path": _path(change.old), "hash": _hash(change.old)}
|
||||||
"path": _path(change.old),
|
|
||||||
"hash": _hash(change.old),
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
elif change.typ == MODIFY:
|
elif change.typ == MODIFY:
|
||||||
ret["modified"].append(
|
ret["modified"].append(
|
||||||
{
|
{
|
||||||
"path": _path(change.old),
|
"path": _path(change.old),
|
||||||
"hash": {
|
"hash": {"old": _hash(change.old), "new": _hash(change.new)},
|
||||||
"old": _hash(change.old),
|
|
||||||
"new": _hash(change.new),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
elif change.typ == RENAME:
|
elif change.typ == RENAME:
|
||||||
ret["renamed"].append(
|
ret["renamed"].append(
|
||||||
{
|
{
|
||||||
"path": {
|
"path": {"old": _path(change.old), "new": _path(change.new)},
|
||||||
"old": _path(change.old),
|
|
||||||
"new": _path(change.new),
|
|
||||||
},
|
|
||||||
"hash": _hash(change.old),
|
"hash": _hash(change.old),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -93,10 +79,7 @@ def _diff(old, new, data_keys, with_missing=False):
|
|||||||
and not old.storage_map.cache_exists(change.old)
|
and not old.storage_map.cache_exists(change.old)
|
||||||
):
|
):
|
||||||
ret["not in cache"].append(
|
ret["not in cache"].append(
|
||||||
{
|
{"path": _path(change.old), "hash": _hash(change.old)}
|
||||||
"path": _path(change.old),
|
|
||||||
"hash": _hash(change.old),
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return ret if any(ret.values()) else {}
|
return ret if any(ret.values()) else {}
|
||||||
@@ -137,11 +120,7 @@ def diff(
|
|||||||
def onerror(target, _exc):
|
def onerror(target, _exc):
|
||||||
missing_targets[rev].add(target) # noqa: B023
|
missing_targets[rev].add(target) # noqa: B023
|
||||||
|
|
||||||
view = self.index.targets_view(
|
view = self.index.targets_view(targets, onerror=onerror, recursive=recursive)
|
||||||
targets,
|
|
||||||
onerror=onerror,
|
|
||||||
recursive=recursive,
|
|
||||||
)
|
|
||||||
|
|
||||||
data_keys.update(view.data_keys.get("repo", set()))
|
data_keys.update(view.data_keys.get("repo", set()))
|
||||||
|
|
||||||
@@ -149,12 +128,7 @@ def diff(
|
|||||||
from .index import build_data_index
|
from .index import build_data_index
|
||||||
|
|
||||||
with ui.status("Building workspace index"):
|
with ui.status("Building workspace index"):
|
||||||
data = build_data_index(
|
data = build_data_index(view, self.root_dir, self.fs, compute_hash=True)
|
||||||
view,
|
|
||||||
self.root_dir,
|
|
||||||
self.fs,
|
|
||||||
compute_hash=True,
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
data = view.data["repo"]
|
data = view.data["repo"]
|
||||||
|
|
||||||
|
|||||||
@@ -131,16 +131,9 @@ class Experiments:
|
|||||||
self._log_reproduced(results, tmp_dir=tmp_dir)
|
self._log_reproduced(results, tmp_dir=tmp_dir)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
def queue_one(
|
def queue_one(self, queue: "BaseStashQueue", **kwargs) -> "QueueEntry":
|
||||||
self,
|
|
||||||
queue: "BaseStashQueue",
|
|
||||||
**kwargs,
|
|
||||||
) -> "QueueEntry":
|
|
||||||
"""Queue a single experiment."""
|
"""Queue a single experiment."""
|
||||||
return self.new(
|
return self.new(queue, **kwargs)
|
||||||
queue,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def reproduce_celery(
|
def reproduce_celery(
|
||||||
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
|
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
|
||||||
@@ -148,16 +141,10 @@ class Experiments:
|
|||||||
results: dict[str, str] = {}
|
results: dict[str, str] = {}
|
||||||
if entries is None:
|
if entries is None:
|
||||||
entries = list(
|
entries = list(
|
||||||
chain(
|
chain(self.celery_queue.iter_active(), self.celery_queue.iter_queued())
|
||||||
self.celery_queue.iter_active(),
|
|
||||||
self.celery_queue.iter_queued(),
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.debug(
|
logger.debug("reproduce all these entries '%s'", entries)
|
||||||
"reproduce all these entries '%s'",
|
|
||||||
entries,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not entries:
|
if not entries:
|
||||||
return results
|
return results
|
||||||
@@ -212,12 +199,7 @@ class Experiments:
|
|||||||
else:
|
else:
|
||||||
ui.write("Experiment results have been applied to your workspace.")
|
ui.write("Experiment results have been applied to your workspace.")
|
||||||
|
|
||||||
def new(
|
def new(self, queue: "BaseStashQueue", *args, **kwargs) -> "QueueEntry":
|
||||||
self,
|
|
||||||
queue: "BaseStashQueue",
|
|
||||||
*args,
|
|
||||||
**kwargs,
|
|
||||||
) -> "QueueEntry":
|
|
||||||
"""Create and enqueue a new experiment.
|
"""Create and enqueue a new experiment.
|
||||||
|
|
||||||
Experiment will be derived from the current workspace.
|
Experiment will be derived from the current workspace.
|
||||||
|
|||||||
@@ -33,10 +33,9 @@ def apply(repo: "Repo", rev: str, **kwargs):
|
|||||||
try:
|
try:
|
||||||
exp_rev = resolve_rev(repo.scm, rev)
|
exp_rev = resolve_rev(repo.scm, rev)
|
||||||
except RevError as exc:
|
except RevError as exc:
|
||||||
(
|
(exp_ref_info, queue_entry) = exps.celery_queue.get_ref_and_entry_by_names(rev)[
|
||||||
exp_ref_info,
|
rev
|
||||||
queue_entry,
|
]
|
||||||
) = exps.celery_queue.get_ref_and_entry_by_names(rev)[rev]
|
|
||||||
if exp_ref_info:
|
if exp_ref_info:
|
||||||
exp_rev = repo.scm.get_ref(str(exp_ref_info))
|
exp_rev = repo.scm.get_ref(str(exp_ref_info))
|
||||||
elif queue_entry:
|
elif queue_entry:
|
||||||
|
|||||||
@@ -10,10 +10,7 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def switch_repo(
|
def switch_repo(repo: "Repo", rev: str) -> Iterator[tuple["Repo", str]]:
|
||||||
repo: "Repo",
|
|
||||||
rev: str,
|
|
||||||
) -> Iterator[tuple["Repo", str]]:
|
|
||||||
"""Return a repo instance (brancher) switched to rev.
|
"""Return a repo instance (brancher) switched to rev.
|
||||||
|
|
||||||
If rev is the name of a running experiment, the returned instance will be
|
If rev is the name of a running experiment, the returned instance will be
|
||||||
|
|||||||
@@ -3,11 +3,7 @@ import os
|
|||||||
from collections.abc import Collection, Iterable, Iterator
|
from collections.abc import Collection, Iterable, Iterator
|
||||||
from dataclasses import fields
|
from dataclasses import fields
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import first
|
from funcy import first
|
||||||
from scmrepo.exceptions import SCMError as InnerSCMError
|
from scmrepo.exceptions import SCMError as InnerSCMError
|
||||||
@@ -58,13 +54,7 @@ def collect_rev(
|
|||||||
else:
|
else:
|
||||||
orig_cwd = None
|
orig_cwd = None
|
||||||
try:
|
try:
|
||||||
data = _collect_rev(
|
data = _collect_rev(repo, rev, param_deps=param_deps, force=force, **kwargs)
|
||||||
repo,
|
|
||||||
rev,
|
|
||||||
param_deps=param_deps,
|
|
||||||
force=force,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
if not (rev == "workspace" or param_deps or data.contains_error):
|
if not (rev == "workspace" or param_deps or data.contains_error):
|
||||||
cache.put(data, force=True)
|
cache.put(data, force=True)
|
||||||
return ExpState(rev=rev, data=data)
|
return ExpState(rev=rev, data=data)
|
||||||
|
|||||||
@@ -51,11 +51,7 @@ class MultipleBranchError(DvcException):
|
|||||||
|
|
||||||
|
|
||||||
class AmbiguousExpRefInfo(InvalidArgumentError):
|
class AmbiguousExpRefInfo(InvalidArgumentError):
|
||||||
def __init__(
|
def __init__(self, exp_name: str, exp_ref_list: Iterable["ExpRefInfo"]):
|
||||||
self,
|
|
||||||
exp_name: str,
|
|
||||||
exp_ref_list: Iterable["ExpRefInfo"],
|
|
||||||
):
|
|
||||||
msg = [
|
msg = [
|
||||||
(
|
(
|
||||||
f"Ambiguous name '{exp_name}' refers to multiple experiments."
|
f"Ambiguous name '{exp_name}' refers to multiple experiments."
|
||||||
|
|||||||
@@ -8,14 +8,7 @@ from contextlib import contextmanager
|
|||||||
from dataclasses import asdict, dataclass
|
from dataclasses import asdict, dataclass
|
||||||
from enum import IntEnum
|
from enum import IntEnum
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Callable,
|
|
||||||
NamedTuple,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from scmrepo.exceptions import SCMError
|
from scmrepo.exceptions import SCMError
|
||||||
|
|
||||||
|
|||||||
@@ -132,11 +132,7 @@ class TempDirExecutor(BaseLocalExecutor):
|
|||||||
self._update_config(repo.config.read("local"))
|
self._update_config(repo.config.read("local"))
|
||||||
|
|
||||||
def _update_config(self, update):
|
def _update_config(self, update):
|
||||||
local_config = os.path.join(
|
local_config = os.path.join(self.root_dir, self.dvc_dir, "config.local")
|
||||||
self.root_dir,
|
|
||||||
self.dvc_dir,
|
|
||||||
"config.local",
|
|
||||||
)
|
|
||||||
logger.debug("Writing experiments local config '%s'", local_config)
|
logger.debug("Writing experiments local config '%s'", local_config)
|
||||||
if os.path.exists(local_config):
|
if os.path.exists(local_config):
|
||||||
conf_obj = ConfigObj(local_config)
|
conf_obj = ConfigObj(local_config)
|
||||||
@@ -188,12 +184,7 @@ class WorkspaceExecutor(BaseLocalExecutor):
|
|||||||
self._detach_stack = ExitStack()
|
self._detach_stack = ExitStack()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_stash_entry(
|
def from_stash_entry(cls, repo: "Repo", entry: "ExpStashEntry", **kwargs):
|
||||||
cls,
|
|
||||||
repo: "Repo",
|
|
||||||
entry: "ExpStashEntry",
|
|
||||||
**kwargs,
|
|
||||||
):
|
|
||||||
root_dir = repo.scm.root_dir
|
root_dir = repo.scm.root_dir
|
||||||
executor: "WorkspaceExecutor" = cls._from_stash_entry(
|
executor: "WorkspaceExecutor" = cls._from_stash_entry(
|
||||||
repo, entry, root_dir, **kwargs
|
repo, entry, root_dir, **kwargs
|
||||||
|
|||||||
@@ -1,10 +1,5 @@
|
|||||||
from collections.abc import Iterable, Mapping
|
from collections.abc import Iterable, Mapping
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import compact, group_by
|
from funcy import compact, group_by
|
||||||
from scmrepo.git.backend.base import SyncStatus
|
from scmrepo.git.backend.base import SyncStatus
|
||||||
|
|||||||
@@ -2,13 +2,7 @@ import os
|
|||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections.abc import Collection, Generator, Iterable, Mapping
|
from collections.abc import Collection, Generator, Iterable, Mapping
|
||||||
from dataclasses import asdict, dataclass
|
from dataclasses import asdict, dataclass
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
NamedTuple,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from dvc_studio_client.post_live_metrics import get_studio_config
|
from dvc_studio_client.post_live_metrics import get_studio_config
|
||||||
from funcy import retry
|
from funcy import retry
|
||||||
@@ -275,12 +269,7 @@ class BaseStashQueue(ABC):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def logs(
|
def logs(self, rev: str, encoding: Optional[str] = None, follow: bool = False):
|
||||||
self,
|
|
||||||
rev: str,
|
|
||||||
encoding: Optional[str] = None,
|
|
||||||
follow: bool = False,
|
|
||||||
):
|
|
||||||
"""Print redirected output logs for an exp process.
|
"""Print redirected output logs for an exp process.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -336,9 +325,7 @@ class BaseStashQueue(ABC):
|
|||||||
self._stash_commit_deps(*args, **kwargs)
|
self._stash_commit_deps(*args, **kwargs)
|
||||||
|
|
||||||
# save additional repro command line arguments
|
# save additional repro command line arguments
|
||||||
run_env = {
|
run_env = {DVC_EXP_BASELINE_REV: baseline_rev}
|
||||||
DVC_EXP_BASELINE_REV: baseline_rev,
|
|
||||||
}
|
|
||||||
if not name:
|
if not name:
|
||||||
name = get_random_exp_name(self.scm, baseline_rev)
|
name = get_random_exp_name(self.scm, baseline_rev)
|
||||||
run_env[DVC_EXP_NAME] = name
|
run_env[DVC_EXP_NAME] = name
|
||||||
@@ -502,10 +489,7 @@ class BaseStashQueue(ABC):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@retry(180, errors=LockError, timeout=1)
|
@retry(180, errors=LockError, timeout=1)
|
||||||
def get_stash_entry(
|
def get_stash_entry(exp: "Experiments", queue_entry: QueueEntry) -> "ExpStashEntry":
|
||||||
exp: "Experiments",
|
|
||||||
queue_entry: QueueEntry,
|
|
||||||
) -> "ExpStashEntry":
|
|
||||||
stash = ExpStash(exp.scm, queue_entry.stash_ref)
|
stash = ExpStash(exp.scm, queue_entry.stash_ref)
|
||||||
stash_rev = queue_entry.stash_rev
|
stash_rev = queue_entry.stash_rev
|
||||||
with get_exp_rwlock(exp.repo, writes=[queue_entry.stash_ref]):
|
with get_exp_rwlock(exp.repo, writes=[queue_entry.stash_ref]):
|
||||||
|
|||||||
@@ -4,12 +4,7 @@ import logging
|
|||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Collection, Generator, Mapping
|
from collections.abc import Collection, Generator, Mapping
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, NamedTuple, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
NamedTuple,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from celery.result import AsyncResult
|
from celery.result import AsyncResult
|
||||||
from funcy import first
|
from funcy import first
|
||||||
@@ -83,10 +78,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
"dvc-exp-local",
|
"dvc-exp-local",
|
||||||
wdir=self.wdir,
|
wdir=self.wdir,
|
||||||
mkdir=True,
|
mkdir=True,
|
||||||
include=[
|
include=["dvc.repo.experiments.queue.tasks", "dvc_task.proc.tasks"],
|
||||||
"dvc.repo.experiments.queue.tasks",
|
|
||||||
"dvc_task.proc.tasks",
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
app.conf.update({"task_acks_late": True, "result_expires": None})
|
app.conf.update({"task_acks_late": True, "result_expires": None})
|
||||||
return app
|
return app
|
||||||
@@ -379,10 +371,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
raise CannotKillTasksError(remained_revs)
|
raise CannotKillTasksError(remained_revs)
|
||||||
|
|
||||||
def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None:
|
def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None:
|
||||||
logger.debug(
|
logger.debug("Found active tasks: '%s' to kill", list(entries.values()))
|
||||||
"Found active tasks: '%s' to kill",
|
|
||||||
list(entries.values()),
|
|
||||||
)
|
|
||||||
inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks(
|
inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks(
|
||||||
entries, force
|
entries, force
|
||||||
)
|
)
|
||||||
@@ -418,20 +407,11 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
if to_kill:
|
if to_kill:
|
||||||
self._kill_entries(to_kill, True)
|
self._kill_entries(to_kill, True)
|
||||||
|
|
||||||
def follow(
|
def follow(self, entry: QueueEntry, encoding: Optional[str] = None):
|
||||||
self,
|
|
||||||
entry: QueueEntry,
|
|
||||||
encoding: Optional[str] = None,
|
|
||||||
):
|
|
||||||
for line in self.proc.follow(entry.stash_rev, encoding):
|
for line in self.proc.follow(entry.stash_rev, encoding):
|
||||||
ui.write(line, end="")
|
ui.write(line, end="")
|
||||||
|
|
||||||
def logs(
|
def logs(self, rev: str, encoding: Optional[str] = None, follow: bool = False):
|
||||||
self,
|
|
||||||
rev: str,
|
|
||||||
encoding: Optional[str] = None,
|
|
||||||
follow: bool = False,
|
|
||||||
):
|
|
||||||
queue_entry: Optional[QueueEntry] = self.match_queue_entry_by_name(
|
queue_entry: Optional[QueueEntry] = self.match_queue_entry_by_name(
|
||||||
{rev}, self.iter_active(), self.iter_done()
|
{rev}, self.iter_active(), self.iter_done()
|
||||||
).get(rev)
|
).get(rev)
|
||||||
@@ -458,8 +438,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
f"No output logs found for experiment '{rev}'"
|
f"No output logs found for experiment '{rev}'"
|
||||||
)
|
)
|
||||||
with open(
|
with open(
|
||||||
proc_info.stdout,
|
proc_info.stdout, encoding=encoding or locale.getpreferredencoding()
|
||||||
encoding=encoding or locale.getpreferredencoding(),
|
|
||||||
) as fobj:
|
) as fobj:
|
||||||
ui.write(fobj.read())
|
ui.write(fobj.read())
|
||||||
|
|
||||||
@@ -560,9 +539,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
def collect_queued_data(
|
def collect_queued_data(
|
||||||
self,
|
self, baseline_revs: Optional[Collection[str]], **kwargs
|
||||||
baseline_revs: Optional[Collection[str]],
|
|
||||||
**kwargs,
|
|
||||||
) -> dict[str, list["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
from dvc.repo.experiments.collect import collect_rev
|
from dvc.repo.experiments.collect import collect_rev
|
||||||
from dvc.repo.experiments.serialize import (
|
from dvc.repo.experiments.serialize import (
|
||||||
|
|||||||
@@ -37,20 +37,14 @@ def remove_tasks( # noqa: C901, PLR0912
|
|||||||
failed_stash_revs.append(failed_rev_all[entry.stash_rev])
|
failed_stash_revs.append(failed_rev_all[entry.stash_rev])
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for (
|
for msg, queue_entry in celery_queue._iter_queued():
|
||||||
msg,
|
|
||||||
queue_entry,
|
|
||||||
) in celery_queue._iter_queued():
|
|
||||||
if queue_entry.stash_rev in stash_revs and msg.delivery_tag:
|
if queue_entry.stash_rev in stash_revs and msg.delivery_tag:
|
||||||
celery_queue.celery.reject(msg.delivery_tag)
|
celery_queue.celery.reject(msg.delivery_tag)
|
||||||
finally:
|
finally:
|
||||||
celery_queue.stash.remove_revs(list(stash_revs.values()))
|
celery_queue.stash.remove_revs(list(stash_revs.values()))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for (
|
for msg, queue_entry in celery_queue._iter_processed():
|
||||||
msg,
|
|
||||||
queue_entry,
|
|
||||||
) in celery_queue._iter_processed():
|
|
||||||
if queue_entry not in done_entry_set:
|
if queue_entry not in done_entry_set:
|
||||||
continue
|
continue
|
||||||
task_id = msg.headers["id"]
|
task_id = msg.headers["id"]
|
||||||
@@ -115,10 +109,7 @@ def celery_clear(
|
|||||||
return removed
|
return removed
|
||||||
|
|
||||||
|
|
||||||
def celery_remove(
|
def celery_remove(self: "LocalCeleryQueue", revs: Collection[str]) -> list[str]:
|
||||||
self: "LocalCeleryQueue",
|
|
||||||
revs: Collection[str],
|
|
||||||
) -> list[str]:
|
|
||||||
"""Remove the specified entries from the queue.
|
"""Remove the specified entries from the queue.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
|
|||||||
@@ -23,11 +23,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]:
|
|||||||
remote_url : remote executor's url
|
remote_url : remote executor's url
|
||||||
"""
|
"""
|
||||||
refs = []
|
refs = []
|
||||||
for ref in iter_remote_refs(
|
for ref in iter_remote_refs(scm, remote_url, base=EXPS_NAMESPACE):
|
||||||
scm,
|
|
||||||
remote_url,
|
|
||||||
base=EXPS_NAMESPACE,
|
|
||||||
):
|
|
||||||
if not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH:
|
if not ref.startswith(EXEC_NAMESPACE) and ref != EXPS_STASH:
|
||||||
refs.append(ref)
|
refs.append(ref)
|
||||||
return refs
|
return refs
|
||||||
|
|||||||
@@ -182,12 +182,7 @@ class WorkspaceQueue(BaseStashQueue):
|
|||||||
def shutdown(self, kill: bool = False):
|
def shutdown(self, kill: bool = False):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def logs(
|
def logs(self, rev: str, encoding: Optional[str] = None, follow: bool = False):
|
||||||
self,
|
|
||||||
rev: str,
|
|
||||||
encoding: Optional[str] = None,
|
|
||||||
follow: bool = False,
|
|
||||||
):
|
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def get_running_exp(self) -> Optional[str]:
|
def get_running_exp(self) -> Optional[str]:
|
||||||
|
|||||||
@@ -1,14 +1,7 @@
|
|||||||
from collections import Counter, defaultdict
|
from collections import Counter, defaultdict
|
||||||
from collections.abc import Iterable, Iterator, Mapping
|
from collections.abc import Iterable, Iterator, Mapping
|
||||||
from datetime import date, datetime
|
from datetime import date, datetime
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Literal,
|
|
||||||
NamedTuple,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from dvc.exceptions import InvalidArgumentError
|
from dvc.exceptions import InvalidArgumentError
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
|
|||||||
@@ -118,20 +118,12 @@ class ApplyStash(Stash):
|
|||||||
m = self.MESSAGE_RE.match(msg)
|
m = self.MESSAGE_RE.match(msg)
|
||||||
if m:
|
if m:
|
||||||
revs[entry.new_sha.decode("utf-8")] = ApplyStashEntry(
|
revs[entry.new_sha.decode("utf-8")] = ApplyStashEntry(
|
||||||
i,
|
i, m.group("head_rev"), m.group("rev"), m.group("name")
|
||||||
m.group("head_rev"),
|
|
||||||
m.group("rev"),
|
|
||||||
m.group("name"),
|
|
||||||
)
|
)
|
||||||
return revs
|
return revs
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def format_message(
|
def format_message(cls, head_rev: str, rev: str, name: Optional[str] = None) -> str:
|
||||||
cls,
|
|
||||||
head_rev: str,
|
|
||||||
rev: str,
|
|
||||||
name: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
return cls.MESSAGE_FORMAT.format(
|
return cls.MESSAGE_FORMAT.format(
|
||||||
head_rev=head_rev, rev=rev, name=name if name else ""
|
head_rev=head_rev, rev=rev, name=name if name else ""
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -4,12 +4,7 @@ import sys
|
|||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Generator, Iterable, Mapping
|
from collections.abc import Generator, Iterable, Mapping
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Callable, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Callable,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from dvc.exceptions import InvalidArgumentError
|
from dvc.exceptions import InvalidArgumentError
|
||||||
from dvc.repo.experiments.exceptions import AmbiguousExpRefInfo
|
from dvc.repo.experiments.exceptions import AmbiguousExpRefInfo
|
||||||
@@ -282,12 +277,7 @@ def fetch_all_exps(scm: "Git", url: str, progress: Optional[Callable] = None, **
|
|||||||
for ref in iter_remote_refs(scm, url, base=EXPS_NAMESPACE)
|
for ref in iter_remote_refs(scm, url, base=EXPS_NAMESPACE)
|
||||||
if not _ignore_ref(ref)
|
if not _ignore_ref(ref)
|
||||||
]
|
]
|
||||||
scm.fetch_refspecs(
|
scm.fetch_refspecs(url, refspecs, progress=progress, **kwargs)
|
||||||
url,
|
|
||||||
refspecs,
|
|
||||||
progress=progress,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def gen_random_name():
|
def gen_random_name():
|
||||||
|
|||||||
+1
-5
@@ -163,11 +163,7 @@ def fetch( # noqa: PLR0913
|
|||||||
tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())),
|
tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())),
|
||||||
)
|
)
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(desc="Collecting", unit="entry", leave=True) as pb:
|
||||||
desc="Collecting",
|
|
||||||
unit="entry",
|
|
||||||
leave=True,
|
|
||||||
) as pb:
|
|
||||||
data = collect(
|
data = collect(
|
||||||
[idx.data["repo"] for idx in indexes.values()],
|
[idx.data["repo"] for idx in indexes.values()],
|
||||||
"remote",
|
"remote",
|
||||||
|
|||||||
+1
-6
@@ -61,9 +61,4 @@ def get(
|
|||||||
else:
|
else:
|
||||||
fs = repo.dvcfs
|
fs = repo.dvcfs
|
||||||
fs_path = fs.from_os_path(path)
|
fs_path = fs.from_os_path(path)
|
||||||
download(
|
download(fs, fs_path, os.path.abspath(out), jobs=jobs)
|
||||||
fs,
|
|
||||||
fs_path,
|
|
||||||
os.path.abspath(out),
|
|
||||||
jobs=jobs,
|
|
||||||
)
|
|
||||||
|
|||||||
+2
-12
@@ -4,14 +4,7 @@ from collections import defaultdict
|
|||||||
from collections.abc import Iterable, Iterator
|
from collections.abc import Iterable, Iterator
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Callable,
|
|
||||||
NamedTuple,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy.debug import format_time
|
from funcy.debug import format_time
|
||||||
|
|
||||||
@@ -577,10 +570,7 @@ class Index:
|
|||||||
return by_workspace
|
return by_workspace
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _hash_targets(
|
def _hash_targets(targets: Iterable[Optional[str]], **kwargs: Any) -> int:
|
||||||
targets: Iterable[Optional[str]],
|
|
||||||
**kwargs: Any,
|
|
||||||
) -> int:
|
|
||||||
return hash(
|
return hash(
|
||||||
(
|
(
|
||||||
frozenset(targets),
|
frozenset(targets),
|
||||||
|
|||||||
+1
-4
@@ -84,10 +84,7 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False): # noqa:
|
|||||||
proj = Repo(root_dir)
|
proj = Repo(root_dir)
|
||||||
|
|
||||||
with proj.scm_context(autostage=True) as context:
|
with proj.scm_context(autostage=True) as context:
|
||||||
files = [
|
files = [config.files["repo"], dvcignore]
|
||||||
config.files["repo"],
|
|
||||||
dvcignore,
|
|
||||||
]
|
|
||||||
ignore_file = context.scm.ignore_file
|
ignore_file = context.scm.ignore_file
|
||||||
if ignore_file:
|
if ignore_file:
|
||||||
files.extend([os.path.join(dvc_dir, ignore_file)])
|
files.extend([os.path.join(dvc_dir, ignore_file)])
|
||||||
|
|||||||
@@ -1,13 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
from collections.abc import Iterable, Iterator
|
from collections.abc import Iterable, Iterator
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Optional,
|
|
||||||
TypedDict,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import ldistinct
|
from funcy import ldistinct
|
||||||
from scmrepo.exceptions import SCMError
|
from scmrepo.exceptions import SCMError
|
||||||
|
|||||||
@@ -18,11 +18,7 @@ logger = logger.getChild(__name__)
|
|||||||
|
|
||||||
|
|
||||||
@map_scm_exception()
|
@map_scm_exception()
|
||||||
def _external_repo(
|
def _external_repo(url, rev: Optional[str] = None, **kwargs) -> "Repo":
|
||||||
url,
|
|
||||||
rev: Optional[str] = None,
|
|
||||||
**kwargs,
|
|
||||||
) -> "Repo":
|
|
||||||
logger.debug("Creating external repo %s@%s", url, rev)
|
logger.debug("Creating external repo %s@%s", url, rev)
|
||||||
path = _cached_clone(url, rev)
|
path = _cached_clone(url, rev)
|
||||||
# Local HEAD points to the tip of whatever branch we first cloned from
|
# Local HEAD points to the tip of whatever branch we first cloned from
|
||||||
|
|||||||
@@ -6,13 +6,7 @@ from collections.abc import Iterator
|
|||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from multiprocessing import cpu_count
|
from multiprocessing import cpu_count
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Callable,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
import dpath
|
import dpath
|
||||||
import dpath.options
|
import dpath.options
|
||||||
@@ -415,10 +409,7 @@ def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = No
|
|||||||
onerror=onerror,
|
onerror=onerror,
|
||||||
)
|
)
|
||||||
|
|
||||||
dpath.merge(
|
dpath.merge(result, {"": unpacked})
|
||||||
result,
|
|
||||||
{"": unpacked},
|
|
||||||
)
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@@ -458,15 +449,9 @@ def _resolve_definitions(
|
|||||||
data_path = _normpath(fs.join(config_dir, plot_id))
|
data_path = _normpath(fs.join(config_dir, plot_id))
|
||||||
if _matches(targets, config_path, plot_id):
|
if _matches(targets, config_path, plot_id):
|
||||||
unpacked = unpack_if_dir(
|
unpacked = unpack_if_dir(
|
||||||
fs,
|
fs, data_path, props={**plot_props, **props}, onerror=onerror
|
||||||
data_path,
|
|
||||||
props={**plot_props, **props},
|
|
||||||
onerror=onerror,
|
|
||||||
)
|
|
||||||
dpath.merge(
|
|
||||||
result,
|
|
||||||
unpacked,
|
|
||||||
)
|
)
|
||||||
|
dpath.merge(result, unpacked)
|
||||||
elif _matches(targets, config_path, plot_id):
|
elif _matches(targets, config_path, plot_id):
|
||||||
adjusted_props = _adjust_sources(fs, plot_props, config_dir)
|
adjusted_props = _adjust_sources(fs, plot_props, config_dir)
|
||||||
dpath.merge(result, {"data": {plot_id: {**adjusted_props, **props}}})
|
dpath.merge(result, {"data": {plot_id: {**adjusted_props, **props}}})
|
||||||
@@ -489,17 +474,9 @@ def _collect_pipeline_files(repo, targets: list[str], props, onerror=None):
|
|||||||
dvcfile_defs_dict[k] = v
|
dvcfile_defs_dict[k] = v
|
||||||
|
|
||||||
resolved = _resolve_definitions(
|
resolved = _resolve_definitions(
|
||||||
repo.dvcfs,
|
repo.dvcfs, targets, props, dvcfile_path, dvcfile_defs_dict, onerror=onerror
|
||||||
targets,
|
|
||||||
props,
|
|
||||||
dvcfile_path,
|
|
||||||
dvcfile_defs_dict,
|
|
||||||
onerror=onerror,
|
|
||||||
)
|
|
||||||
dpath.merge(
|
|
||||||
result,
|
|
||||||
{dvcfile_path: resolved},
|
|
||||||
)
|
)
|
||||||
|
dpath.merge(result, {dvcfile_path: resolved})
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@@ -515,15 +492,9 @@ def _collect_definitions(
|
|||||||
props = props or {}
|
props = props or {}
|
||||||
|
|
||||||
fs = repo.dvcfs
|
fs = repo.dvcfs
|
||||||
dpath.merge(
|
dpath.merge(result, _collect_pipeline_files(repo, targets, props, onerror=onerror))
|
||||||
result,
|
|
||||||
_collect_pipeline_files(repo, targets, props, onerror=onerror),
|
|
||||||
)
|
|
||||||
|
|
||||||
dpath.merge(
|
dpath.merge(result, _collect_output_plots(repo, targets, props, onerror=onerror))
|
||||||
result,
|
|
||||||
_collect_output_plots(repo, targets, props, onerror=onerror),
|
|
||||||
)
|
|
||||||
|
|
||||||
for target in targets:
|
for target in targets:
|
||||||
if not result or fs.exists(target):
|
if not result or fs.exists(target):
|
||||||
|
|||||||
+1
-5
@@ -120,11 +120,7 @@ def push( # noqa: PLR0913
|
|||||||
tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())),
|
tokenize(sorted(idx.data_tree.hash_info.value for idx in indexes.values())),
|
||||||
)
|
)
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(desc="Collecting", unit="entry", leave=True) as pb:
|
||||||
desc="Collecting",
|
|
||||||
unit="entry",
|
|
||||||
leave=True,
|
|
||||||
) as pb:
|
|
||||||
data = collect(
|
data = collect(
|
||||||
[idx.data["repo"] for idx in indexes.values()],
|
[idx.data["repo"] for idx in indexes.values()],
|
||||||
"remote",
|
"remote",
|
||||||
|
|||||||
@@ -1,13 +1,5 @@
|
|||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Callable, NoReturn, Optional, TypeVar, Union, cast
|
||||||
TYPE_CHECKING,
|
|
||||||
Callable,
|
|
||||||
NoReturn,
|
|
||||||
Optional,
|
|
||||||
TypeVar,
|
|
||||||
Union,
|
|
||||||
cast,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import ldistinct
|
from funcy import ldistinct
|
||||||
|
|
||||||
|
|||||||
@@ -3,12 +3,7 @@ import shlex
|
|||||||
from collections.abc import Iterable, Iterator
|
from collections.abc import Iterable, Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
from dvc.utils import relpath
|
from dvc.utils import relpath
|
||||||
|
|||||||
+1
-6
@@ -402,12 +402,7 @@ class StageLoad:
|
|||||||
from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound
|
from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound
|
||||||
|
|
||||||
try:
|
try:
|
||||||
stages = self.collect(
|
stages = self.collect(target, with_deps, recursive, graph)
|
||||||
target,
|
|
||||||
with_deps,
|
|
||||||
recursive,
|
|
||||||
graph,
|
|
||||||
)
|
|
||||||
except StageFileDoesNotExistError as exc:
|
except StageFileDoesNotExistError as exc:
|
||||||
# collect() might try to use `target` as a stage name
|
# collect() might try to use `target` as a stage name
|
||||||
# and throw error that dvc.yaml does not exist, whereas it
|
# and throw error that dvc.yaml does not exist, whereas it
|
||||||
|
|||||||
+1
-4
@@ -61,10 +61,7 @@ def update( # noqa: C901
|
|||||||
)
|
)
|
||||||
if to_remote:
|
if to_remote:
|
||||||
raise InvalidArgumentError("--to-remote can't be used with worktree update")
|
raise InvalidArgumentError("--to-remote can't be used with worktree update")
|
||||||
update_worktree_stages(
|
update_worktree_stages(self, other_stage_infos)
|
||||||
self,
|
|
||||||
other_stage_infos,
|
|
||||||
)
|
|
||||||
|
|
||||||
stages = import_stages | {stage_info.stage for stage_info in other_stage_infos}
|
stages = import_stages | {stage_info.stage for stage_info in other_stage_infos}
|
||||||
return list(stages)
|
return list(stages)
|
||||||
|
|||||||
+7
-32
@@ -162,20 +162,13 @@ def _merge_push_meta( # noqa: C901
|
|||||||
out.meta.remote = remote
|
out.meta.remote = remote
|
||||||
|
|
||||||
|
|
||||||
def update_worktree_stages(
|
def update_worktree_stages(repo: "Repo", stage_infos: Iterable["StageInfo"]):
|
||||||
repo: "Repo",
|
|
||||||
stage_infos: Iterable["StageInfo"],
|
|
||||||
):
|
|
||||||
from dvc.repo.index import IndexView
|
from dvc.repo.index import IndexView
|
||||||
|
|
||||||
def outs_filter(out: "Output") -> bool:
|
def outs_filter(out: "Output") -> bool:
|
||||||
return out.is_in_repo and out.use_cache and out.can_push
|
return out.is_in_repo and out.use_cache and out.can_push
|
||||||
|
|
||||||
view = IndexView(
|
view = IndexView(repo.index, stage_infos, outs_filter=outs_filter)
|
||||||
repo.index,
|
|
||||||
stage_infos,
|
|
||||||
outs_filter=outs_filter,
|
|
||||||
)
|
|
||||||
local_index = view.data["repo"]
|
local_index = view.data["repo"]
|
||||||
remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {}
|
remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {}
|
||||||
for stage in view.stages:
|
for stage in view.stages:
|
||||||
@@ -194,10 +187,7 @@ def _update_worktree_out(
|
|||||||
|
|
||||||
remote_name = out.remote or out.meta.remote
|
remote_name = out.remote or out.meta.remote
|
||||||
if not remote_name:
|
if not remote_name:
|
||||||
logger.warning(
|
logger.warning("Could not update '%s', it was never pushed to a remote", out)
|
||||||
"Could not update '%s', it was never pushed to a remote",
|
|
||||||
out,
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
if remote_name in remote_indexes:
|
if remote_name in remote_indexes:
|
||||||
@@ -211,10 +201,7 @@ def _update_worktree_out(
|
|||||||
remote_indexes[remote_name] = remote, remote_index
|
remote_indexes[remote_name] = remote, remote_index
|
||||||
_workspace, key = out.index_key
|
_workspace, key = out.index_key
|
||||||
if key not in remote_index:
|
if key not in remote_index:
|
||||||
logger.warning(
|
logger.warning("Could not update '%s', it does not exist in the remote", out)
|
||||||
"Could not update '%s', it does not exist in the remote",
|
|
||||||
out,
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
entry = remote_index[key]
|
entry = remote_index[key]
|
||||||
@@ -226,20 +213,11 @@ def _update_worktree_out(
|
|||||||
for subkey, subentry in remote_index.iteritems(key)
|
for subkey, subentry in remote_index.iteritems(key)
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
logger.warning(
|
logger.warning("Could not update '%s', directory is empty in the remote", out)
|
||||||
"Could not update '%s', directory is empty in the remote",
|
|
||||||
out,
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
_fetch_out_changes(out, local_index, remote_index, remote)
|
_fetch_out_changes(out, local_index, remote_index, remote)
|
||||||
_update_out_meta(
|
_update_out_meta(repo, out, local_index, remote_index, remote)
|
||||||
repo,
|
|
||||||
out,
|
|
||||||
local_index,
|
|
||||||
remote_index,
|
|
||||||
remote,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _fetch_out_changes(
|
def _fetch_out_changes(
|
||||||
@@ -253,10 +231,7 @@ def _fetch_out_changes(
|
|||||||
|
|
||||||
old, new = _get_diff_indexes(out, local_index, remote_index)
|
old, new = _get_diff_indexes(out, local_index, remote_index)
|
||||||
|
|
||||||
with TqdmCallback(
|
with TqdmCallback(unit="entry", desc="Comparing indexes") as cb:
|
||||||
unit="entry",
|
|
||||||
desc="Comparing indexes",
|
|
||||||
) as cb:
|
|
||||||
diff = compare(
|
diff = compare(
|
||||||
old,
|
old,
|
||||||
new,
|
new,
|
||||||
|
|||||||
+1
-7
@@ -3,13 +3,7 @@ import os
|
|||||||
from collections.abc import Iterator, Mapping
|
from collections.abc import Iterator, Mapping
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Literal, Optional, Union, overload
|
||||||
TYPE_CHECKING,
|
|
||||||
Literal,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
overload,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import group_by
|
from funcy import group_by
|
||||||
from scmrepo.base import Base # noqa: F401
|
from scmrepo.base import Base # noqa: F401
|
||||||
|
|||||||
+2
-12
@@ -4,13 +4,7 @@ from collections import defaultdict
|
|||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Optional,
|
|
||||||
TypeVar,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import project
|
from funcy import project
|
||||||
|
|
||||||
@@ -664,11 +658,7 @@ class Stage(params.StageParams):
|
|||||||
return stats
|
return stats
|
||||||
|
|
||||||
for out in self.filter_outs(kwargs.get("filter_info")):
|
for out in self.filter_outs(kwargs.get("filter_info")):
|
||||||
key, outs = self._checkout(
|
key, outs = self._checkout(out, allow_missing=allow_missing, **kwargs)
|
||||||
out,
|
|
||||||
allow_missing=allow_missing,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
if key:
|
if key:
|
||||||
stats[key].extend(outs)
|
stats[key].extend(outs)
|
||||||
return stats
|
return stats
|
||||||
|
|||||||
+1
-4
@@ -266,10 +266,7 @@ class StageCache:
|
|||||||
|
|
||||||
src_name = from_fs.name(src)
|
src_name = from_fs.name(src)
|
||||||
parent_name = from_fs.name(from_fs.parent(src))
|
parent_name = from_fs.name(from_fs.parent(src))
|
||||||
with TqdmCallback(
|
with TqdmCallback(desc=src_name, bytes=True) as cb:
|
||||||
desc=src_name,
|
|
||||||
bytes=True,
|
|
||||||
) as cb:
|
|
||||||
func(from_fs, src, to_fs, dst, callback=cb)
|
func(from_fs, src, to_fs, dst, callback=cb)
|
||||||
ret.append((parent_name, src_name))
|
ret.append((parent_name, src_name))
|
||||||
return ret
|
return ret
|
||||||
|
|||||||
+2
-11
@@ -44,13 +44,7 @@ def update_import(
|
|||||||
stage.frozen = frozen
|
stage.frozen = frozen
|
||||||
|
|
||||||
|
|
||||||
def sync_import(
|
def sync_import(stage, dry=False, force=False, jobs=None, no_download=False):
|
||||||
stage,
|
|
||||||
dry=False,
|
|
||||||
force=False,
|
|
||||||
jobs=None,
|
|
||||||
no_download=False,
|
|
||||||
):
|
|
||||||
"""Synchronize import's outs to the workspace."""
|
"""Synchronize import's outs to the workspace."""
|
||||||
logger.info("Importing '%s' -> '%s'", stage.deps[0], stage.outs[0])
|
logger.info("Importing '%s' -> '%s'", stage.deps[0], stage.outs[0])
|
||||||
if dry:
|
if dry:
|
||||||
@@ -64,7 +58,4 @@ def sync_import(
|
|||||||
if stage.is_repo_import or stage.is_db_import:
|
if stage.is_repo_import or stage.is_db_import:
|
||||||
stage.deps[0].update()
|
stage.deps[0].update()
|
||||||
else:
|
else:
|
||||||
stage.deps[0].download(
|
stage.deps[0].download(stage.outs[0], jobs=jobs)
|
||||||
stage.outs[0],
|
|
||||||
jobs=jobs,
|
|
||||||
)
|
|
||||||
|
|||||||
+2
-9
@@ -25,12 +25,7 @@ logger = logger.getChild(__name__)
|
|||||||
|
|
||||||
|
|
||||||
class StageLoader(Mapping):
|
class StageLoader(Mapping):
|
||||||
def __init__(
|
def __init__(self, dvcfile: "ProjectFile", data, lockfile_data=None):
|
||||||
self,
|
|
||||||
dvcfile: "ProjectFile",
|
|
||||||
data,
|
|
||||||
lockfile_data=None,
|
|
||||||
):
|
|
||||||
self.dvcfile = dvcfile
|
self.dvcfile = dvcfile
|
||||||
self.resolver = self.dvcfile.resolver
|
self.resolver = self.dvcfile.resolver
|
||||||
self.data = data or {}
|
self.data = data or {}
|
||||||
@@ -190,9 +185,7 @@ class SingleStageLoader(Mapping):
|
|||||||
def __getitem__(self, item):
|
def __getitem__(self, item):
|
||||||
if item:
|
if item:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Ignoring name '%s' for single stage in '%s'.",
|
"Ignoring name '%s' for single stage in '%s'.", item, self.dvcfile
|
||||||
item,
|
|
||||||
self.dvcfile,
|
|
||||||
)
|
)
|
||||||
# during `load`, we remove attributes from stage data, so as to
|
# during `load`, we remove attributes from stage data, so as to
|
||||||
# not duplicate, therefore, for MappingView, we need to deepcopy.
|
# not duplicate, therefore, for MappingView, we need to deepcopy.
|
||||||
|
|||||||
+1
-7
@@ -119,13 +119,7 @@ def _pull_missing_deps(stage):
|
|||||||
stage.repo.pull(dep.def_path)
|
stage.repo.pull(dep.def_path)
|
||||||
|
|
||||||
|
|
||||||
def run_stage(
|
def run_stage(stage, dry=False, force=False, run_env=None, **kwargs):
|
||||||
stage,
|
|
||||||
dry=False,
|
|
||||||
force=False,
|
|
||||||
run_env=None,
|
|
||||||
**kwargs,
|
|
||||||
):
|
|
||||||
if not force:
|
if not force:
|
||||||
if kwargs.get("pull") and not dry:
|
if kwargs.get("pull") and not dry:
|
||||||
_pull_missing_deps(stage)
|
_pull_missing_deps(stage)
|
||||||
|
|||||||
@@ -1,13 +1,7 @@
|
|||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from operator import attrgetter
|
from operator import attrgetter
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Optional, Union, no_type_check
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Optional,
|
|
||||||
Union,
|
|
||||||
no_type_check,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import post_processing
|
from funcy import post_processing
|
||||||
|
|
||||||
|
|||||||
@@ -15,12 +15,7 @@ class TestAPI:
|
|||||||
assert api.get_url("foo") == expected_url
|
assert api.get_url("foo") == expected_url
|
||||||
|
|
||||||
def test_open(self, tmp_dir, dvc, remote):
|
def test_open(self, tmp_dir, dvc, remote):
|
||||||
tmp_dir.dvc_gen(
|
tmp_dir.dvc_gen({"foo": "foo-text", "dir": {"bar": "bar-text"}})
|
||||||
{
|
|
||||||
"foo": "foo-text",
|
|
||||||
"dir": {"bar": "bar-text"},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
dvc.push()
|
dvc.push()
|
||||||
|
|
||||||
# Remove cache to force download
|
# Remove cache to force download
|
||||||
|
|||||||
@@ -5,13 +5,7 @@ import pytest
|
|||||||
pytestmark = pytest.mark.requires(minversion=(2, 15, 0), reason="new command")
|
pytestmark = pytest.mark.requires(minversion=(2, 15, 0), reason="new command")
|
||||||
|
|
||||||
|
|
||||||
def test_data_status(
|
def test_data_status(bench_dvc, tmp_dir, scm, dvc, make_dataset):
|
||||||
bench_dvc,
|
|
||||||
tmp_dir,
|
|
||||||
scm,
|
|
||||||
dvc,
|
|
||||||
make_dataset,
|
|
||||||
):
|
|
||||||
args = ("data", "status")
|
args = ("data", "status")
|
||||||
dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
|
dataset = make_dataset(cache=True, files=True, dvcfile=True, commit=False)
|
||||||
rmtree(dvc.tmp_dir)
|
rmtree(dvc.tmp_dir)
|
||||||
|
|||||||
@@ -86,11 +86,7 @@ def pytest_addoption(parser):
|
|||||||
help="Path or url to dvc-bench git repo (for loading benchmark datasets)",
|
help="Path or url to dvc-bench git repo (for loading benchmark datasets)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.addoption(
|
parser.addoption("--project-rev", type=str, help="Project revision to test")
|
||||||
"--project-rev",
|
|
||||||
type=str,
|
|
||||||
help="Project revision to test",
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.addoption(
|
parser.addoption(
|
||||||
"--project-git-repo",
|
"--project-git-repo",
|
||||||
|
|||||||
@@ -246,11 +246,7 @@ def docker_services(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
cmd = "docker-compose version"
|
cmd = "docker-compose version"
|
||||||
subprocess.check_output(
|
subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) # noqa: S602
|
||||||
cmd,
|
|
||||||
stderr=subprocess.STDOUT,
|
|
||||||
shell=True, # noqa: S602
|
|
||||||
)
|
|
||||||
except subprocess.CalledProcessError as err:
|
except subprocess.CalledProcessError as err:
|
||||||
out = (err.output or b"").decode("utf-8")
|
out = (err.output or b"").decode("utf-8")
|
||||||
pytest.skip(f"docker-compose is not installed: {out}")
|
pytest.skip(f"docker-compose is not installed: {out}")
|
||||||
|
|||||||
@@ -209,9 +209,7 @@ class URLInfo(_BasePath):
|
|||||||
return self._spath
|
return self._spath
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def _path(
|
def _path(self) -> "_URLPathInfo":
|
||||||
self,
|
|
||||||
) -> "_URLPathInfo":
|
|
||||||
return _URLPathInfo(self._spath)
|
return _URLPathInfo(self._spath)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|||||||
@@ -20,7 +20,5 @@ def _add_script(tmp_dir, path, contents=""):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def copy_script(
|
def copy_script(tmp_dir):
|
||||||
tmp_dir,
|
|
||||||
):
|
|
||||||
return _add_script(tmp_dir, "copy.py", COPY_SCRIPT)
|
return _add_script(tmp_dir, "copy.py", COPY_SCRIPT)
|
||||||
|
|||||||
@@ -173,9 +173,7 @@ class TestImportURLVersionAware:
|
|||||||
dvc.cache.local.clear()
|
dvc.cache.local.clear()
|
||||||
remove(tmp_dir / "data_dir")
|
remove(tmp_dir / "data_dir")
|
||||||
assert dvc.pull()["fetched"] == 1
|
assert dvc.pull()["fetched"] == 1
|
||||||
assert (tmp_dir / "data_dir").read_text() == {
|
assert (tmp_dir / "data_dir").read_text() == {"subdir": {"file": "file"}}
|
||||||
"subdir": {"file": "file"},
|
|
||||||
}
|
|
||||||
|
|
||||||
dvc.commit(force=True)
|
dvc.commit(force=True)
|
||||||
assert dvc.status() == {}
|
assert dvc.status() == {}
|
||||||
@@ -193,11 +191,7 @@ class TestLsUrl:
|
|||||||
cloud.gen({fname: "foo contents"})
|
cloud.gen({fname: "foo contents"})
|
||||||
fs, fs_path = parse_external_url(cloud.url, cloud.config)
|
fs, fs_path = parse_external_url(cloud.url, cloud.config)
|
||||||
result = ls_url(str(cloud / fname), fs_config=cloud.config)
|
result = ls_url(str(cloud / fname), fs_config=cloud.config)
|
||||||
match_files(
|
match_files(fs, result, [{"path": fs.join(fs_path, fname), "isdir": False}])
|
||||||
fs,
|
|
||||||
result,
|
|
||||||
[{"path": fs.join(fs_path, fname), "isdir": False}],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_dir(self, cloud):
|
def test_dir(self, cloud):
|
||||||
cloud.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"})
|
cloud.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"})
|
||||||
|
|||||||
+1
-8
@@ -1,13 +1,6 @@
|
|||||||
from collections.abc import Iterable, Iterator, Sequence
|
from collections.abc import Iterable, Iterator, Sequence
|
||||||
from contextlib import contextmanager, nullcontext
|
from contextlib import contextmanager, nullcontext
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Callable, Optional, TextIO, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Callable,
|
|
||||||
Optional,
|
|
||||||
TextIO,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
import colorama
|
import colorama
|
||||||
|
|
||||||
|
|||||||
@@ -17,9 +17,7 @@ def parse_params(path_params: Iterable[str]) -> list[dict[str, list[str]]]:
|
|||||||
return [{path: params} for path, params in ret.items()]
|
return [{path: params} for path, params in ret.items()]
|
||||||
|
|
||||||
|
|
||||||
def to_path_overrides(
|
def to_path_overrides(path_params: Iterable[str]) -> dict[str, list[str]]:
|
||||||
path_params: Iterable[str],
|
|
||||||
) -> dict[str, list[str]]:
|
|
||||||
"""Group overrides by path"""
|
"""Group overrides by path"""
|
||||||
from dvc.dependency.param import ParamsDependency
|
from dvc.dependency.param import ParamsDependency
|
||||||
|
|
||||||
|
|||||||
@@ -1,15 +1,7 @@
|
|||||||
"""Common utilities for serialize."""
|
"""Common utilities for serialize."""
|
||||||
import os
|
import os
|
||||||
from contextlib import AbstractContextManager, contextmanager
|
from contextlib import AbstractContextManager, contextmanager
|
||||||
from typing import (
|
from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol, TextIO, Union
|
||||||
TYPE_CHECKING,
|
|
||||||
Any,
|
|
||||||
Callable,
|
|
||||||
Optional,
|
|
||||||
Protocol,
|
|
||||||
TextIO,
|
|
||||||
Union,
|
|
||||||
)
|
|
||||||
|
|
||||||
from funcy import reraise
|
from funcy import reraise
|
||||||
|
|
||||||
|
|||||||
@@ -31,38 +31,15 @@ def params_repo(tmp_dir, scm, dvc):
|
|||||||
tmp_dir.gen("params.json", '{"bar": 2, "foobar": 3}')
|
tmp_dir.gen("params.json", '{"bar": 2, "foobar": 3}')
|
||||||
tmp_dir.gen("other_params.json", '{"foo": {"bar": 4}}')
|
tmp_dir.gen("other_params.json", '{"foo": {"bar": 4}}')
|
||||||
|
|
||||||
dvc.run(
|
dvc.run(name="stage-0", cmd="echo stage-0")
|
||||||
name="stage-0",
|
|
||||||
cmd="echo stage-0",
|
|
||||||
)
|
|
||||||
|
|
||||||
dvc.run(
|
dvc.run(name="stage-1", cmd="echo stage-1", params=["foo", "params.json:bar"])
|
||||||
name="stage-1",
|
|
||||||
cmd="echo stage-1",
|
|
||||||
params=["foo", "params.json:bar"],
|
|
||||||
)
|
|
||||||
|
|
||||||
dvc.run(
|
dvc.run(name="stage-2", cmd="echo stage-2", params=["other_params.json:foo"])
|
||||||
name="stage-2",
|
|
||||||
cmd="echo stage-2",
|
|
||||||
params=["other_params.json:foo"],
|
|
||||||
)
|
|
||||||
|
|
||||||
dvc.run(
|
dvc.run(name="stage-3", cmd="echo stage-2", params=["params.json:foobar"])
|
||||||
name="stage-3",
|
|
||||||
cmd="echo stage-2",
|
|
||||||
params=["params.json:foobar"],
|
|
||||||
)
|
|
||||||
|
|
||||||
scm.add(
|
scm.add(["params.yaml", "params.json", "other_params.json", "dvc.yaml", "dvc.lock"])
|
||||||
[
|
|
||||||
"params.yaml",
|
|
||||||
"params.json",
|
|
||||||
"other_params.json",
|
|
||||||
"dvc.yaml",
|
|
||||||
"dvc.lock",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
scm.commit("commit dvc files")
|
scm.commit("commit dvc files")
|
||||||
|
|
||||||
tmp_dir.gen("params.yaml", "foo: 5")
|
tmp_dir.gen("params.yaml", "foo: 5")
|
||||||
@@ -77,10 +54,7 @@ def metrics_repo(tmp_dir, scm, dvc, run_copy_metrics):
|
|||||||
scm.commit("prepare data")
|
scm.commit("prepare data")
|
||||||
sub_dir = tmp_dir / "eval"
|
sub_dir = tmp_dir / "eval"
|
||||||
sub_dir.mkdir()
|
sub_dir.mkdir()
|
||||||
tmp_dir.gen(
|
tmp_dir.gen("tmp_train_val_metrics.json", json.dumps(TRAIN_METRICS[0]))
|
||||||
"tmp_train_val_metrics.json",
|
|
||||||
json.dumps(TRAIN_METRICS[0]),
|
|
||||||
)
|
|
||||||
train_metrics_file = os.path.join(sub_dir, "train_val_metrics.json")
|
train_metrics_file = os.path.join(sub_dir, "train_val_metrics.json")
|
||||||
run_copy_metrics(
|
run_copy_metrics(
|
||||||
"tmp_train_val_metrics.json",
|
"tmp_train_val_metrics.json",
|
||||||
@@ -107,10 +81,7 @@ def metrics_repo(tmp_dir, scm, dvc, run_copy_metrics):
|
|||||||
scm.commit("test model")
|
scm.commit("test model")
|
||||||
|
|
||||||
with tmp_dir.branch("better-model", new=True):
|
with tmp_dir.branch("better-model", new=True):
|
||||||
tmp_dir.gen(
|
tmp_dir.gen("tmp_train_val_metrics.json", json.dumps(TRAIN_METRICS[1]))
|
||||||
"tmp_train_val_metrics.json",
|
|
||||||
json.dumps(TRAIN_METRICS[1]),
|
|
||||||
)
|
|
||||||
run_copy_metrics(
|
run_copy_metrics(
|
||||||
"tmp_train_val_metrics.json",
|
"tmp_train_val_metrics.json",
|
||||||
train_metrics_file,
|
train_metrics_file,
|
||||||
@@ -243,11 +214,7 @@ def test_params_show_while_running_stage(tmp_dir, dvc):
|
|||||||
def test_params_show_repo(tmp_dir, erepo_dir):
|
def test_params_show_repo(tmp_dir, erepo_dir):
|
||||||
with erepo_dir.chdir():
|
with erepo_dir.chdir():
|
||||||
erepo_dir.scm_gen("params.yaml", "foo: 1", commit="Create params.yaml")
|
erepo_dir.scm_gen("params.yaml", "foo: 1", commit="Create params.yaml")
|
||||||
erepo_dir.dvc.run(
|
erepo_dir.dvc.run(name="stage-1", cmd="echo stage-1", params=["foo"])
|
||||||
name="stage-1",
|
|
||||||
cmd="echo stage-1",
|
|
||||||
params=["foo"],
|
|
||||||
)
|
|
||||||
assert api.params_show(repo=erepo_dir) == {"foo": 1}
|
assert api.params_show(repo=erepo_dir) == {"foo": 1}
|
||||||
|
|
||||||
|
|
||||||
@@ -268,10 +235,7 @@ def test_params_show_no_params_found(tmp_dir, dvc):
|
|||||||
def test_params_show_stage_without_params(tmp_dir, dvc):
|
def test_params_show_stage_without_params(tmp_dir, dvc):
|
||||||
tmp_dir.gen("params.yaml", "foo: 1")
|
tmp_dir.gen("params.yaml", "foo: 1")
|
||||||
|
|
||||||
dvc.run(
|
dvc.run(name="stage-0", cmd="echo stage-0")
|
||||||
name="stage-0",
|
|
||||||
cmd="echo stage-0",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert api.params_show(stages="stage-0") == {}
|
assert api.params_show(stages="stage-0") == {}
|
||||||
|
|
||||||
|
|||||||
@@ -70,9 +70,7 @@ def test_artifacts_add_subdir(tmp_dir, dvc):
|
|||||||
name: Artifact(**values) for name, values in dvcyaml["artifacts"].items()
|
name: Artifact(**values) for name, values in dvcyaml["artifacts"].items()
|
||||||
}
|
}
|
||||||
artifacts["new"] = new_art
|
artifacts["new"] = new_art
|
||||||
assert tmp_dir.dvc.artifacts.read() == {
|
assert tmp_dir.dvc.artifacts.read() == {f"subdir{os.path.sep}dvc.yaml": artifacts}
|
||||||
f"subdir{os.path.sep}dvc.yaml": artifacts,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_artifacts_add_abspath(tmp_dir, dvc):
|
def test_artifacts_add_abspath(tmp_dir, dvc):
|
||||||
@@ -114,11 +112,7 @@ bad_dvcyaml_extra_field = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bad_dvcyaml_missing_path = {
|
bad_dvcyaml_missing_path = {"artifacts": {"lol": {}}}
|
||||||
"artifacts": {
|
|
||||||
"lol": {},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
@@ -148,16 +142,7 @@ def test_artifacts_read_fails_on_id_duplication(tmp_dir, dvc):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"name",
|
"name", ["1", "m", "nn", "m1", "1nn", "model-prod", "model-prod-v1"]
|
||||||
[
|
|
||||||
"1",
|
|
||||||
"m",
|
|
||||||
"nn",
|
|
||||||
"m1",
|
|
||||||
"1nn",
|
|
||||||
"model-prod",
|
|
||||||
"model-prod-v1",
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
def test_name_is_compatible(name):
|
def test_name_is_compatible(name):
|
||||||
check_name_format(name)
|
check_name_format(name)
|
||||||
|
|||||||
@@ -34,10 +34,6 @@ def params_repo(tmp_dir, scm, dvc):
|
|||||||
(tmp_dir / "params.yaml").dump(
|
(tmp_dir / "params.yaml").dump(
|
||||||
{"foo": [{"bar": 1}, {"baz": 2}], "goo": {"bag": 3.0}, "lorem": False}
|
{"foo": [{"bar": 1}, {"baz": 2}], "goo": {"bag": 3.0}, "lorem": False}
|
||||||
)
|
)
|
||||||
dvc.run(
|
dvc.run(cmd="echo foo", params=["params.yaml:"], name="foo")
|
||||||
cmd="echo foo",
|
|
||||||
params=["params.yaml:"],
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml"])
|
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml"])
|
||||||
scm.commit("init")
|
scm.commit("init")
|
||||||
|
|||||||
@@ -46,11 +46,7 @@ def test_apply_failed(tmp_dir, scm, dvc, failed_exp_stage, mocker):
|
|||||||
),
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
mocker.patch.object(
|
mocker.patch.object(queue, "iter_queued", return_value=[])
|
||||||
queue,
|
|
||||||
"iter_queued",
|
|
||||||
return_value=[],
|
|
||||||
)
|
|
||||||
|
|
||||||
dvc.experiments.apply(exp_rev)
|
dvc.experiments.apply(exp_rev)
|
||||||
assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 3"
|
assert (tmp_dir / "params.yaml").read_text().strip() == "foo: 3"
|
||||||
|
|||||||
@@ -100,14 +100,7 @@ def test_file_permissions(tmp_dir, scm, dvc, exp_stage, mocker):
|
|||||||
assert stat.S_IMODE(os.stat(tmp_dir / "copy.py").st_mode) == mode
|
assert stat.S_IMODE(os.stat(tmp_dir / "copy.py").st_mode) == mode
|
||||||
|
|
||||||
|
|
||||||
def test_failed_exp_workspace(
|
def test_failed_exp_workspace(tmp_dir, scm, dvc, failed_exp_stage, mocker, capsys):
|
||||||
tmp_dir,
|
|
||||||
scm,
|
|
||||||
dvc,
|
|
||||||
failed_exp_stage,
|
|
||||||
mocker,
|
|
||||||
capsys,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("params.yaml", "foo: 2")
|
tmp_dir.gen("params.yaml", "foo: 2")
|
||||||
with pytest.raises(ReproductionError):
|
with pytest.raises(ReproductionError):
|
||||||
dvc.experiments.run(failed_exp_stage.addressing)
|
dvc.experiments.run(failed_exp_stage.addressing)
|
||||||
@@ -445,13 +438,7 @@ def test_subrepo(tmp_dir, request, scm, workspace):
|
|||||||
name="copy-file",
|
name="copy-file",
|
||||||
no_exec=True,
|
no_exec=True,
|
||||||
)
|
)
|
||||||
scm.add(
|
scm.add([subrepo / "dvc.yaml", subrepo / "copy.py", subrepo / "params.yaml"])
|
||||||
[
|
|
||||||
subrepo / "dvc.yaml",
|
|
||||||
subrepo / "copy.py",
|
|
||||||
subrepo / "params.yaml",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
scm.commit("init")
|
scm.commit("init")
|
||||||
|
|
||||||
results = subrepo.dvc.experiments.run(
|
results = subrepo.dvc.experiments.run(
|
||||||
@@ -593,11 +580,7 @@ def test_experiment_name_invalid(tmp_dir, scm, dvc, exp_stage, mocker):
|
|||||||
|
|
||||||
new_mock = mocker.spy(BaseStashQueue, "_stash_exp")
|
new_mock = mocker.spy(BaseStashQueue, "_stash_exp")
|
||||||
with pytest.raises(InvalidArgumentError):
|
with pytest.raises(InvalidArgumentError):
|
||||||
dvc.experiments.run(
|
dvc.experiments.run(exp_stage.addressing, name="fo^o", params=["foo=3"])
|
||||||
exp_stage.addressing,
|
|
||||||
name="fo^o",
|
|
||||||
params=["foo=3"],
|
|
||||||
)
|
|
||||||
new_mock.assert_not_called()
|
new_mock.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
@@ -651,10 +634,7 @@ def test_run_env(tmp_dir, dvc, scm, mocker):
|
|||||||
)
|
)
|
||||||
(tmp_dir / "dump_run_env.py").write_text(dump_run_env)
|
(tmp_dir / "dump_run_env.py").write_text(dump_run_env)
|
||||||
baseline = scm.get_rev()
|
baseline = scm.get_rev()
|
||||||
dvc.stage.add(
|
dvc.stage.add(cmd="python dump_run_env.py", name="run_env")
|
||||||
cmd="python dump_run_env.py",
|
|
||||||
name="run_env",
|
|
||||||
)
|
|
||||||
dvc.experiments.run()
|
dvc.experiments.run()
|
||||||
assert (tmp_dir / DVC_EXP_BASELINE_REV).read_text().strip() == baseline
|
assert (tmp_dir / DVC_EXP_BASELINE_REV).read_text().strip() == baseline
|
||||||
assert (tmp_dir / DVC_EXP_NAME).read_text().strip()
|
assert (tmp_dir / DVC_EXP_NAME).read_text().strip()
|
||||||
@@ -728,11 +708,7 @@ def test_local_config_is_propagated_to_tmp(tmp_dir, scm, dvc):
|
|||||||
@pytest.mark.parametrize("tmp", [True, False])
|
@pytest.mark.parametrize("tmp", [True, False])
|
||||||
def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
|
def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
|
||||||
(tmp_dir / "dvc.yaml").dump(
|
(tmp_dir / "dvc.yaml").dump(
|
||||||
{
|
{"metrics": ["metrics.json"], "params": ["params.yaml"], "plots": ["plots.csv"]}
|
||||||
"metrics": ["metrics.json"],
|
|
||||||
"params": ["params.yaml"],
|
|
||||||
"plots": ["plots.csv"],
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(
|
||||||
cmd="touch metrics.json && touch params.yaml && touch plots.csv",
|
cmd="touch metrics.json && touch params.yaml && touch plots.csv",
|
||||||
@@ -748,10 +724,7 @@ def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
|
|||||||
|
|
||||||
@pytest.mark.parametrize("tmp", [True, False])
|
@pytest.mark.parametrize("tmp", [True, False])
|
||||||
def test_copy_paths(tmp_dir, scm, dvc, tmp):
|
def test_copy_paths(tmp_dir, scm, dvc, tmp):
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(cmd="cat file && ls dir", name="foo")
|
||||||
cmd="cat file && ls dir",
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
||||||
|
|
||||||
(tmp_dir / "dir").mkdir()
|
(tmp_dir / "dir").mkdir()
|
||||||
@@ -770,10 +743,7 @@ def test_copy_paths(tmp_dir, scm, dvc, tmp):
|
|||||||
|
|
||||||
|
|
||||||
def test_copy_paths_errors(tmp_dir, scm, dvc, mocker):
|
def test_copy_paths_errors(tmp_dir, scm, dvc, mocker):
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(cmd="echo foo", name="foo")
|
||||||
cmd="echo foo",
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
||||||
|
|
||||||
with pytest.raises(DvcException, match="Unable to copy"):
|
with pytest.raises(DvcException, match="Unable to copy"):
|
||||||
@@ -814,10 +784,7 @@ def test_mixed_git_dvc_out(tmp_dir, scm, dvc, exp_stage):
|
|||||||
|
|
||||||
@pytest.mark.parametrize("tmp", [True, False])
|
@pytest.mark.parametrize("tmp", [True, False])
|
||||||
def test_custom_commit_message(tmp_dir, scm, dvc, tmp):
|
def test_custom_commit_message(tmp_dir, scm, dvc, tmp):
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(cmd="echo foo", name="foo")
|
||||||
cmd="echo foo",
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
||||||
|
|
||||||
exp = first(
|
exp = first(
|
||||||
|
|||||||
@@ -10,15 +10,7 @@ def to_dict(tasks):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("follow", [True, False])
|
@pytest.mark.parametrize("follow", [True, False])
|
||||||
def test_celery_logs(
|
def test_celery_logs(tmp_dir, scm, dvc, failed_exp_stage, follow, capsys, test_queue):
|
||||||
tmp_dir,
|
|
||||||
scm,
|
|
||||||
dvc,
|
|
||||||
failed_exp_stage,
|
|
||||||
follow,
|
|
||||||
capsys,
|
|
||||||
test_queue,
|
|
||||||
):
|
|
||||||
celery_queue = dvc.experiments.celery_queue
|
celery_queue = dvc.experiments.celery_queue
|
||||||
dvc.experiments.run(failed_exp_stage.addressing, queue=True, name="foo")
|
dvc.experiments.run(failed_exp_stage.addressing, queue=True, name="foo")
|
||||||
dvc.experiments.run(run_all=True)
|
dvc.experiments.run(run_all=True)
|
||||||
@@ -37,23 +29,14 @@ def test_queue_doesnt_remove_untracked_params_file(tmp_dir, dvc, scm):
|
|||||||
"""Regression test for https://github.com/iterative/dvc/issues/7842"""
|
"""Regression test for https://github.com/iterative/dvc/issues/7842"""
|
||||||
tmp_dir.gen("params.yaml", "foo: 1")
|
tmp_dir.gen("params.yaml", "foo: 1")
|
||||||
stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
|
stage = dvc.run(cmd="echo ${foo}", params=["foo"], name="echo-foo")
|
||||||
scm.add(
|
scm.add(["dvc.yaml", "dvc.lock", ".gitignore"])
|
||||||
[
|
|
||||||
"dvc.yaml",
|
|
||||||
"dvc.lock",
|
|
||||||
".gitignore",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
scm.commit("init")
|
scm.commit("init")
|
||||||
dvc.experiments.run(stage.addressing, params=["foo=2"], queue=True)
|
dvc.experiments.run(stage.addressing, params=["foo=2"], queue=True)
|
||||||
assert (tmp_dir / "params.yaml").exists()
|
assert (tmp_dir / "params.yaml").exists()
|
||||||
|
|
||||||
|
|
||||||
def test_copy_paths_queue(tmp_dir, scm, dvc):
|
def test_copy_paths_queue(tmp_dir, scm, dvc):
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(cmd="cat file && ls dir", name="foo")
|
||||||
cmd="cat file && ls dir",
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
||||||
|
|
||||||
(tmp_dir / "dir").mkdir()
|
(tmp_dir / "dir").mkdir()
|
||||||
@@ -72,10 +55,7 @@ def test_copy_paths_queue(tmp_dir, scm, dvc):
|
|||||||
|
|
||||||
|
|
||||||
def test_custom_commit_message_queue(tmp_dir, scm, dvc):
|
def test_custom_commit_message_queue(tmp_dir, scm, dvc):
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(cmd="echo foo", name="foo")
|
||||||
cmd="echo foo",
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
||||||
|
|
||||||
dvc.experiments.run(stage.addressing, queue=True, message="custom commit message")
|
dvc.experiments.run(stage.addressing, queue=True, message="custom commit message")
|
||||||
|
|||||||
@@ -356,11 +356,7 @@ def test_get(tmp_dir, scm, dvc, exp_stage, erepo_dir, use_ref):
|
|||||||
exp_ref = first(exp_refs_by_rev(scm, exp_rev))
|
exp_ref = first(exp_refs_by_rev(scm, exp_rev))
|
||||||
|
|
||||||
with erepo_dir.chdir():
|
with erepo_dir.chdir():
|
||||||
Repo.get(
|
Repo.get(str(tmp_dir), "params.yaml", rev=exp_ref.name if use_ref else exp_rev)
|
||||||
str(tmp_dir),
|
|
||||||
"params.yaml",
|
|
||||||
rev=exp_ref.name if use_ref else exp_rev,
|
|
||||||
)
|
|
||||||
assert (erepo_dir / "params.yaml").read_text().strip() == "foo: 2"
|
assert (erepo_dir / "params.yaml").read_text().strip() == "foo: 2"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -121,11 +121,7 @@ def test_exp_save_include_untracked_warning(tmp_dir, dvc, scm, mocker):
|
|||||||
|
|
||||||
def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
|
def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
|
||||||
(tmp_dir / "dvc.yaml").dump(
|
(tmp_dir / "dvc.yaml").dump(
|
||||||
{
|
{"metrics": ["metrics.json"], "params": ["params.yaml"], "plots": ["plots.csv"]}
|
||||||
"metrics": ["metrics.json"],
|
|
||||||
"params": ["params.yaml"],
|
|
||||||
"plots": ["plots.csv"],
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(
|
||||||
cmd="touch metrics.json && touch params.yaml && touch plots.csv",
|
cmd="touch metrics.json && touch params.yaml && touch plots.csv",
|
||||||
@@ -140,10 +136,7 @@ def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc):
|
|||||||
|
|
||||||
|
|
||||||
def test_untracked_dvclock_is_included_in_exp(tmp_dir, scm, dvc):
|
def test_untracked_dvclock_is_included_in_exp(tmp_dir, scm, dvc):
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(cmd="echo foo", name="foo")
|
||||||
cmd="echo foo",
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
||||||
dvc.reproduce(stage.addressing)
|
dvc.reproduce(stage.addressing)
|
||||||
|
|
||||||
|
|||||||
@@ -103,11 +103,7 @@ def test_hydra_sweep(
|
|||||||
patched = mocker.patch.object(dvc.experiments, "queue_one")
|
patched = mocker.patch.object(dvc.experiments, "queue_one")
|
||||||
|
|
||||||
if hydra_enabled:
|
if hydra_enabled:
|
||||||
hydra_setup(
|
hydra_setup(tmp_dir, config_dir="conf", config_name="config")
|
||||||
tmp_dir,
|
|
||||||
config_dir="conf",
|
|
||||||
config_name="config",
|
|
||||||
)
|
|
||||||
with dvc.config.edit() as conf:
|
with dvc.config.edit() as conf:
|
||||||
conf["hydra"]["enabled"] = True
|
conf["hydra"]["enabled"] = True
|
||||||
|
|
||||||
|
|||||||
@@ -49,11 +49,7 @@ def make_executor(local=None, **kwargs):
|
|||||||
local_executor.update(local)
|
local_executor.update(local)
|
||||||
else:
|
else:
|
||||||
local_executor = ANY
|
local_executor = ANY
|
||||||
data = {
|
data = {"state": ANY, "local": local_executor, "name": ANY}
|
||||||
"state": ANY,
|
|
||||||
"local": local_executor,
|
|
||||||
"name": ANY,
|
|
||||||
}
|
|
||||||
data.update(kwargs)
|
data.update(kwargs)
|
||||||
return data
|
return data
|
||||||
|
|
||||||
@@ -62,13 +58,7 @@ def make_data(params=None, **kwargs):
|
|||||||
params = {"data": params or {"foo": 1}}
|
params = {"data": params or {"foo": 1}}
|
||||||
data = {
|
data = {
|
||||||
"rev": ANY,
|
"rev": ANY,
|
||||||
"deps": {
|
"deps": {"copy.py": {"hash": ANY, "size": ANY, "nfiles": None}},
|
||||||
"copy.py": {
|
|
||||||
"hash": ANY,
|
|
||||||
"size": ANY,
|
|
||||||
"nfiles": None,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"metrics": {"metrics.yaml": params},
|
"metrics": {"metrics.yaml": params},
|
||||||
"outs": {},
|
"outs": {},
|
||||||
"params": {"params.yaml": params},
|
"params": {"params.yaml": params},
|
||||||
@@ -233,23 +223,14 @@ def test_show_failed_experiment(tmp_dir, scm, dvc, failed_exp_stage, test_queue)
|
|||||||
"experiments": None,
|
"experiments": None,
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"executor": make_executor(
|
"executor": make_executor(state="failed", local={"returncode": 255}),
|
||||||
state="failed",
|
|
||||||
local={"returncode": 255},
|
|
||||||
),
|
|
||||||
"name": ANY,
|
"name": ANY,
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_show_filter(
|
def test_show_filter(tmp_dir, scm, dvc, capsys, copy_script):
|
||||||
tmp_dir,
|
|
||||||
scm,
|
|
||||||
dvc,
|
|
||||||
capsys,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
capsys.readouterr()
|
capsys.readouterr()
|
||||||
|
|
||||||
params_file = tmp_dir / "params.yaml"
|
params_file = tmp_dir / "params.yaml"
|
||||||
@@ -487,10 +468,7 @@ def test_show_csv(tmp_dir, scm, dvc, exp_stage, capsys):
|
|||||||
|
|
||||||
def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script):
|
def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script):
|
||||||
params_file = tmp_dir / "params.yaml"
|
params_file = tmp_dir / "params.yaml"
|
||||||
params_data = {
|
params_data = {"foo": 1, "goobar": 1}
|
||||||
"foo": 1,
|
|
||||||
"goobar": 1,
|
|
||||||
}
|
|
||||||
(tmp_dir / params_file).dump(params_data)
|
(tmp_dir / params_file).dump(params_data)
|
||||||
|
|
||||||
dvc.run(
|
dvc.run(
|
||||||
@@ -534,10 +512,7 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script):
|
|||||||
@pytest.mark.vscode
|
@pytest.mark.vscode
|
||||||
def test_show_outs(tmp_dir, dvc, scm, erepo_dir, copy_script):
|
def test_show_outs(tmp_dir, dvc, scm, erepo_dir, copy_script):
|
||||||
params_file = tmp_dir / "params.yaml"
|
params_file = tmp_dir / "params.yaml"
|
||||||
params_data = {
|
params_data = {"foo": 1, "bar": 1}
|
||||||
"foo": 1,
|
|
||||||
"bar": 1,
|
|
||||||
}
|
|
||||||
(tmp_dir / params_file).dump(params_data)
|
(tmp_dir / params_file).dump(params_data)
|
||||||
|
|
||||||
dvc.run(
|
dvc.run(
|
||||||
@@ -641,9 +616,7 @@ def test_show_outs(tmp_dir, dvc, scm, erepo_dir, copy_script):
|
|||||||
|
|
||||||
def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script):
|
def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script):
|
||||||
params_file = tmp_dir / "params.yaml"
|
params_file = tmp_dir / "params.yaml"
|
||||||
params_data = {
|
params_data = {"foo": 1}
|
||||||
"foo": 1,
|
|
||||||
}
|
|
||||||
(tmp_dir / params_file).dump(params_data)
|
(tmp_dir / params_file).dump(params_data)
|
||||||
|
|
||||||
dvc.run(
|
dvc.run(
|
||||||
@@ -674,14 +647,7 @@ def test_metrics_renaming(tmp_dir, dvc, scm, capsys, copy_script):
|
|||||||
name="copy-file",
|
name="copy-file",
|
||||||
deps=["copy.py"],
|
deps=["copy.py"],
|
||||||
)
|
)
|
||||||
scm.add(
|
scm.add(["dvc.yaml", "dvc.lock", "params.yaml", "scores.yaml"])
|
||||||
[
|
|
||||||
"dvc.yaml",
|
|
||||||
"dvc.lock",
|
|
||||||
"params.yaml",
|
|
||||||
"scores.yaml",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
scm.commit("scores.yaml")
|
scm.commit("scores.yaml")
|
||||||
scores_rev = scm.get_rev()
|
scores_rev = scm.get_rev()
|
||||||
|
|
||||||
@@ -709,11 +675,7 @@ def test_show_sorted_deps(tmp_dir, dvc, scm, capsys):
|
|||||||
tmp_dir.gen("c", "c")
|
tmp_dir.gen("c", "c")
|
||||||
tmp_dir.gen("z", "z")
|
tmp_dir.gen("z", "z")
|
||||||
|
|
||||||
dvc.run(
|
dvc.run(cmd="echo foo", name="deps", deps=["a", "b", "z", "c"])
|
||||||
cmd="echo foo",
|
|
||||||
name="deps",
|
|
||||||
deps=["a", "b", "z", "c"],
|
|
||||||
)
|
|
||||||
|
|
||||||
capsys.readouterr()
|
capsys.readouterr()
|
||||||
assert main(["exp", "show", "--csv"]) == 0
|
assert main(["exp", "show", "--csv"]) == 0
|
||||||
|
|||||||
@@ -9,11 +9,7 @@ from dvc.exceptions import ReproductionError
|
|||||||
@pytest.mark.parametrize("staged", [True, False])
|
@pytest.mark.parametrize("staged", [True, False])
|
||||||
def test_deleted(tmp_dir, scm, dvc, tmp, staged):
|
def test_deleted(tmp_dir, scm, dvc, tmp, staged):
|
||||||
tmp_dir.scm_gen("file", "file", commit="commit file")
|
tmp_dir.scm_gen("file", "file", commit="commit file")
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(cmd="cat file", deps=["file"], name="foo")
|
||||||
cmd="cat file",
|
|
||||||
deps=["file"],
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
||||||
|
|
||||||
file = tmp_dir / "file"
|
file = tmp_dir / "file"
|
||||||
@@ -33,10 +29,7 @@ def test_deleted(tmp_dir, scm, dvc, tmp, staged):
|
|||||||
@pytest.mark.parametrize("staged", [True, False])
|
@pytest.mark.parametrize("staged", [True, False])
|
||||||
def test_modified(tmp_dir, scm, dvc, caplog, tmp, staged):
|
def test_modified(tmp_dir, scm, dvc, caplog, tmp, staged):
|
||||||
tmp_dir.scm_gen("file", "file", commit="commit file")
|
tmp_dir.scm_gen("file", "file", commit="commit file")
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(cmd="cat file", name="foo")
|
||||||
cmd="cat file",
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
||||||
|
|
||||||
(tmp_dir / "file").write_text("modified_file")
|
(tmp_dir / "file").write_text("modified_file")
|
||||||
@@ -52,10 +45,7 @@ def test_modified(tmp_dir, scm, dvc, caplog, tmp, staged):
|
|||||||
|
|
||||||
@pytest.mark.parametrize("tmp", [True, False])
|
@pytest.mark.parametrize("tmp", [True, False])
|
||||||
def test_staged_new_file(tmp_dir, scm, dvc, tmp):
|
def test_staged_new_file(tmp_dir, scm, dvc, tmp):
|
||||||
stage = dvc.stage.add(
|
stage = dvc.stage.add(cmd="cat file", name="foo")
|
||||||
cmd="cat file",
|
|
||||||
name="foo",
|
|
||||||
)
|
|
||||||
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
|
||||||
|
|
||||||
(tmp_dir / "file").write_text("file")
|
(tmp_dir / "file").write_text("file")
|
||||||
|
|||||||
@@ -3,16 +3,7 @@ from funcy import first
|
|||||||
|
|
||||||
def test_generate_random_exp_name(tmp_dir, dvc, scm, exp_stage, mocker):
|
def test_generate_random_exp_name(tmp_dir, dvc, scm, exp_stage, mocker):
|
||||||
mocked_generator = mocker.MagicMock()
|
mocked_generator = mocker.MagicMock()
|
||||||
mocked_generator.choice.side_effect = [
|
mocked_generator.choice.side_effect = [0, 0, 0, 0, 1, 1, 0, 0]
|
||||||
0,
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
1,
|
|
||||||
1,
|
|
||||||
0,
|
|
||||||
0,
|
|
||||||
]
|
|
||||||
mocker.patch(
|
mocker.patch(
|
||||||
"dvc.repo.experiments.utils.random.Random", return_value=mocked_generator
|
"dvc.repo.experiments.utils.random.Random", return_value=mocked_generator
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -160,13 +160,7 @@ def test_missing_cache(tmp_dir, dvc, run_copy_metrics):
|
|||||||
result = dvc.metrics.show()
|
result = dvc.metrics.show()
|
||||||
metrics2 = result[""]["data"].pop("metrics2.yaml")
|
metrics2 = result[""]["data"].pop("metrics2.yaml")
|
||||||
assert isinstance(metrics2["error"], FileNotFoundError)
|
assert isinstance(metrics2["error"], FileNotFoundError)
|
||||||
assert result == {
|
assert result == {"": {"data": {"metrics.yaml": {"data": 1.1}}}}
|
||||||
"": {
|
|
||||||
"data": {
|
|
||||||
"metrics.yaml": {"data": 1.1},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("use_dvc", [True, False])
|
@pytest.mark.parametrize("use_dvc", [True, False])
|
||||||
@@ -276,10 +270,7 @@ def test_metrics_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
|
|||||||
)
|
)
|
||||||
with (tmp_dir / "dvc.yaml").modify() as d:
|
with (tmp_dir / "dvc.yaml").modify() as d:
|
||||||
# trying to make an output overlaps error
|
# trying to make an output overlaps error
|
||||||
d["stages"]["corrupted-stage"] = {
|
d["stages"]["corrupted-stage"] = {"cmd": "mkdir data", "outs": ["data"]}
|
||||||
"cmd": "mkdir data",
|
|
||||||
"outs": ["data"],
|
|
||||||
}
|
|
||||||
|
|
||||||
# running by clearing and not clearing stuffs
|
# running by clearing and not clearing stuffs
|
||||||
# so as it works even for optimized cases
|
# so as it works even for optimized cases
|
||||||
|
|||||||
@@ -238,11 +238,7 @@ def test_diff_targeted(tmp_dir, scm, dvc, run_copy):
|
|||||||
def test_diff_without_targets_specified(tmp_dir, dvc, scm, file):
|
def test_diff_without_targets_specified(tmp_dir, dvc, scm, file):
|
||||||
params_file = tmp_dir / file
|
params_file = tmp_dir / file
|
||||||
params_file.dump({"foo": {"bar": "bar"}, "x": "0"})
|
params_file.dump({"foo": {"bar": "bar"}, "x": "0"})
|
||||||
dvc.stage.add(
|
dvc.stage.add(name="test", cmd=f"echo {file}", params=[{file: None}])
|
||||||
name="test",
|
|
||||||
cmd=f"echo {file}",
|
|
||||||
params=[{file: None}],
|
|
||||||
)
|
|
||||||
scm.add_commit([params_file, "dvc.yaml"], message="foo")
|
scm.add_commit([params_file, "dvc.yaml"], message="foo")
|
||||||
|
|
||||||
params_file.dump({"foo": {"bar": "baz"}, "y": "100"})
|
params_file.dump({"foo": {"bar": "baz"}, "y": "100"})
|
||||||
|
|||||||
@@ -123,11 +123,7 @@ def test_show_without_targets_specified(tmp_dir, dvc, scm, file):
|
|||||||
params_file = tmp_dir / file
|
params_file = tmp_dir / file
|
||||||
data = {"foo": {"bar": "bar"}, "x": "0"}
|
data = {"foo": {"bar": "bar"}, "x": "0"}
|
||||||
params_file.dump(data)
|
params_file.dump(data)
|
||||||
dvc.stage.add(
|
dvc.stage.add(name="test", cmd=f"echo {file}", params=[{file: None}])
|
||||||
name="test",
|
|
||||||
cmd=f"echo {file}",
|
|
||||||
params=[{file: None}],
|
|
||||||
)
|
|
||||||
|
|
||||||
assert dvc.params.show() == {"": {"data": {file: {"data": data}}}}
|
assert dvc.params.show() == {"": {"data": {file: {"data": data}}}}
|
||||||
|
|
||||||
|
|||||||
@@ -71,9 +71,7 @@ def test_load_vars_with_relpath(tmp_dir, scm, dvc):
|
|||||||
revisions = ["HEAD", "workspace"]
|
revisions = ["HEAD", "workspace"]
|
||||||
for rev in dvc.brancher(revs=["HEAD"]):
|
for rev in dvc.brancher(revs=["HEAD"]):
|
||||||
assert rev == revisions.pop()
|
assert rev == revisions.pop()
|
||||||
d = {
|
d = {"vars": [f"../{DEFAULT_PARAMS_FILE}"]}
|
||||||
"vars": [f"../{DEFAULT_PARAMS_FILE}"],
|
|
||||||
}
|
|
||||||
resolver = DataResolver(dvc, "subdir", d)
|
resolver = DataResolver(dvc, "subdir", d)
|
||||||
assert resolver.context == deepcopy(DATA)
|
assert resolver.context == deepcopy(DATA)
|
||||||
|
|
||||||
|
|||||||
@@ -155,10 +155,7 @@ def test_plots_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run):
|
|||||||
)
|
)
|
||||||
with (tmp_dir / "dvc.yaml").modify() as d:
|
with (tmp_dir / "dvc.yaml").modify() as d:
|
||||||
# trying to make an output overlaps error
|
# trying to make an output overlaps error
|
||||||
d["stages"]["corrupted-stage"] = {
|
d["stages"]["corrupted-stage"] = {"cmd": "mkdir data", "outs": ["data"]}
|
||||||
"cmd": "mkdir data",
|
|
||||||
"outs": ["data"],
|
|
||||||
}
|
|
||||||
|
|
||||||
# running by clearing and not clearing stuffs
|
# running by clearing and not clearing stuffs
|
||||||
# so as it works even for optimized cases
|
# so as it works even for optimized cases
|
||||||
@@ -356,12 +353,7 @@ def test_collect_non_existing_dir(tmp_dir, dvc, run_copy_metrics):
|
|||||||
),
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_top_level_plots(
|
def test_top_level_plots(tmp_dir, dvc, plot_config, expected_datafiles):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
plot_config,
|
|
||||||
expected_datafiles,
|
|
||||||
):
|
|
||||||
data = {
|
data = {
|
||||||
"data1.json": [
|
"data1.json": [
|
||||||
{"a": 1, "b": 0.1, "c": 0.01},
|
{"a": 1, "b": 0.1, "c": 0.01},
|
||||||
|
|||||||
+25
-121
@@ -364,10 +364,7 @@ def test_repro_multiple_params(tmp_dir, dvc):
|
|||||||
name="read_params",
|
name="read_params",
|
||||||
deps=["foo"],
|
deps=["foo"],
|
||||||
outs=["bar"],
|
outs=["bar"],
|
||||||
params=[
|
params=["params2.yaml:lists,floats,name", "answer,floats,nested.nested1"],
|
||||||
"params2.yaml:lists,floats,name",
|
|
||||||
"answer,floats,nested.nested1",
|
|
||||||
],
|
|
||||||
cmd="cat params2.yaml params.yaml > bar",
|
cmd="cat params2.yaml params.yaml > bar",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -528,11 +525,7 @@ def test_repro_skip_pull_if_no_run_cache_is_passed(tmp_dir, dvc, mocker, local_r
|
|||||||
assert not spy_pull.called
|
assert not spy_pull.called
|
||||||
|
|
||||||
|
|
||||||
def test_repro_no_commit(
|
def test_repro_no_commit(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
@@ -548,16 +541,9 @@ def test_repro_no_commit(
|
|||||||
assert not os.path.isdir(dvc.cache.local.path)
|
assert not os.path.isdir(dvc.cache.local.path)
|
||||||
|
|
||||||
|
|
||||||
def test_repro_all_pipelines(
|
def test_repro_all_pipelines(mocker, dvc):
|
||||||
mocker,
|
|
||||||
dvc,
|
|
||||||
):
|
|
||||||
stages = [
|
stages = [
|
||||||
dvc.run(
|
dvc.run(outs=["start.txt"], cmd="echo start > start.txt", name="start"),
|
||||||
outs=["start.txt"],
|
|
||||||
cmd="echo start > start.txt",
|
|
||||||
name="start",
|
|
||||||
),
|
|
||||||
dvc.run(
|
dvc.run(
|
||||||
deps=["start.txt"],
|
deps=["start.txt"],
|
||||||
outs=["middle.txt"],
|
outs=["middle.txt"],
|
||||||
@@ -588,10 +574,7 @@ def test_repro_all_pipelines(
|
|||||||
|
|
||||||
|
|
||||||
class TestReproAlreadyCached:
|
class TestReproAlreadyCached:
|
||||||
def test(
|
def test(self, dvc):
|
||||||
self,
|
|
||||||
dvc,
|
|
||||||
):
|
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
always_changed=True,
|
always_changed=True,
|
||||||
deps=[],
|
deps=[],
|
||||||
@@ -604,11 +587,7 @@ class TestReproAlreadyCached:
|
|||||||
|
|
||||||
assert run_out.hash_info != repro_out.hash_info
|
assert run_out.hash_info != repro_out.hash_info
|
||||||
|
|
||||||
def test_force_with_dependencies(
|
def test_force_with_dependencies(self, tmp_dir, dvc):
|
||||||
self,
|
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
):
|
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
name="datetime",
|
name="datetime",
|
||||||
@@ -654,11 +633,7 @@ def test_repro_shell(tmp_dir, monkeypatch, dvc):
|
|||||||
assert (tmp_dir / "shell.txt").read_text().rstrip() == shell
|
assert (tmp_dir / "shell.txt").read_text().rstrip() == shell
|
||||||
|
|
||||||
|
|
||||||
def test_cmd_repro(
|
def test_cmd_repro(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
@@ -679,11 +654,7 @@ def test_cmd_repro(
|
|||||||
assert ret != 0
|
assert ret != 0
|
||||||
|
|
||||||
|
|
||||||
def test_repro_dep_under_dir(
|
def test_repro_dep_under_dir(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("foo", "foo")
|
tmp_dir.gen("foo", "foo")
|
||||||
tmp_dir.dvc_gen("data", {"file": "file", "sub": {"foo": "foo"}})
|
tmp_dir.dvc_gen("data", {"file": "file", "sub": {"foo": "foo"}})
|
||||||
|
|
||||||
@@ -741,11 +712,7 @@ def test_repro_force(tmp_dir, dvc, copy_script):
|
|||||||
assert len(stages) == 2
|
assert len(stages) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_repro_changed_code(
|
def test_repro_changed_code(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
@@ -762,11 +729,7 @@ def test_repro_changed_code(
|
|||||||
assert len(stages) == 1
|
assert len(stages) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_repro_changed_data(
|
def test_repro_changed_data(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
@@ -783,11 +746,7 @@ def test_repro_changed_data(
|
|||||||
assert len(stages) == 2
|
assert len(stages) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_repro_dry(
|
def test_repro_dry(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
@@ -808,11 +767,7 @@ def test_repro_dry(
|
|||||||
assert not filecmp.cmp("file1", "bar", shallow=False)
|
assert not filecmp.cmp("file1", "bar", shallow=False)
|
||||||
|
|
||||||
|
|
||||||
def test_repro_up_to_date(
|
def test_repro_up_to_date(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
@@ -871,11 +826,7 @@ def test_repro_dry_no_exec(tmp_dir, dvc):
|
|||||||
assert ret == 0
|
assert ret == 0
|
||||||
|
|
||||||
|
|
||||||
def test_repro_changed_deep_data(
|
def test_repro_changed_deep_data(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
dvc.run(
|
dvc.run(
|
||||||
@@ -955,11 +906,7 @@ def test_repro_force_downstream_do_not_force_independent_stages(tmp_dir, dvc, ru
|
|||||||
assert dvc.reproduce(force_downstream=True) == [foo1, foo2, cat]
|
assert dvc.reproduce(force_downstream=True) == [foo1, foo2, cat]
|
||||||
|
|
||||||
|
|
||||||
def test_repro_pipeline(
|
def test_repro_pipeline(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
dvc.run(
|
dvc.run(
|
||||||
@@ -978,11 +925,7 @@ def test_repro_pipeline(
|
|||||||
assert len(stages) == 3
|
assert len(stages) == 3
|
||||||
|
|
||||||
|
|
||||||
def test_repro_pipeline_cli(
|
def test_repro_pipeline_cli(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
@@ -995,11 +938,7 @@ def test_repro_pipeline_cli(
|
|||||||
assert ret == 0
|
assert ret == 0
|
||||||
|
|
||||||
|
|
||||||
def test_repro_pipelines(
|
def test_repro_pipelines(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
foo_stage, bar_stage = tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
foo_stage, bar_stage = tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||||
file1_stage = dvc.run(
|
file1_stage = dvc.run(
|
||||||
outs=["file1"],
|
outs=["file1"],
|
||||||
@@ -1021,11 +960,7 @@ def test_repro_pipelines(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_repro_pipelines_cli(
|
def test_repro_pipelines_cli(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||||
dvc.run(
|
dvc.run(
|
||||||
outs=["file1"],
|
outs=["file1"],
|
||||||
@@ -1062,18 +997,11 @@ def test_freeze_non_existing(dvc, target):
|
|||||||
assert ret != 0
|
assert ret != 0
|
||||||
|
|
||||||
|
|
||||||
def test_repro_frozen_callback(
|
def test_repro_frozen_callback(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("foo", "foo")
|
tmp_dir.gen("foo", "foo")
|
||||||
# NOTE: purposefully not specifying deps or outs
|
# NOTE: purposefully not specifying deps or outs
|
||||||
# to create a callback stage.
|
# to create a callback stage.
|
||||||
stage = dvc.run(
|
stage = dvc.run(cmd="python copy.py foo file1", name="copy-FOO-file1")
|
||||||
cmd="python copy.py foo file1",
|
|
||||||
name="copy-FOO-file1",
|
|
||||||
)
|
|
||||||
|
|
||||||
stages = dvc.reproduce(stage.addressing)
|
stages = dvc.reproduce(stage.addressing)
|
||||||
assert len(stages) == 1
|
assert len(stages) == 1
|
||||||
@@ -1087,11 +1015,7 @@ def test_repro_frozen_callback(
|
|||||||
assert len(stages) == 1
|
assert len(stages) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_repro_frozen_unchanged(
|
def test_repro_frozen_unchanged(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
"""
|
"""
|
||||||
Check that freezing/unfreezing doesn't affect stage state
|
Check that freezing/unfreezing doesn't affect stage state
|
||||||
"""
|
"""
|
||||||
@@ -1155,11 +1079,7 @@ def test_repro_metrics_add_unchanged(tmp_dir, dvc, copy_script):
|
|||||||
assert len(stages) == 0
|
assert len(stages) == 0
|
||||||
|
|
||||||
|
|
||||||
def test_repro_phony(
|
def test_repro_phony(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
dvc.run(
|
dvc.run(
|
||||||
@@ -1176,11 +1096,7 @@ def test_repro_phony(
|
|||||||
assert filecmp.cmp("file1", "bar", shallow=False)
|
assert filecmp.cmp("file1", "bar", shallow=False)
|
||||||
|
|
||||||
|
|
||||||
def test_non_existing_output(
|
def test_non_existing_output(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
@@ -1195,11 +1111,7 @@ def test_non_existing_output(
|
|||||||
dvc.reproduce(stage.addressing)
|
dvc.reproduce(stage.addressing)
|
||||||
|
|
||||||
|
|
||||||
def test_repro_data_source(
|
def test_repro_data_source(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen("bar", "bar")
|
tmp_dir.gen("bar", "bar")
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
@@ -1216,11 +1128,7 @@ def test_repro_data_source(
|
|||||||
assert stages[0].outs[0].hash_info.value == file_md5("bar")
|
assert stages[0].outs[0].hash_info.value == file_md5("bar")
|
||||||
|
|
||||||
|
|
||||||
def test_repro_changed_dir(
|
def test_repro_changed_dir(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen({"foo": "foo", "bar": "bar"})
|
tmp_dir.gen({"foo": "foo", "bar": "bar"})
|
||||||
shutil.copyfile("foo", "file")
|
shutil.copyfile("foo", "file")
|
||||||
|
|
||||||
@@ -1241,11 +1149,7 @@ def test_repro_changed_dir(
|
|||||||
assert len(stages) == 1
|
assert len(stages) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_repro_changed_dir_data(
|
def test_repro_changed_dir_data(tmp_dir, dvc, copy_script):
|
||||||
tmp_dir,
|
|
||||||
dvc,
|
|
||||||
copy_script,
|
|
||||||
):
|
|
||||||
tmp_dir.gen({"data": {"foo": "foo"}, "bar": "bar"})
|
tmp_dir.gen({"data": {"foo": "foo"}, "bar": "bar"})
|
||||||
stage = dvc.run(
|
stage = dvc.run(
|
||||||
outs=["dir"],
|
outs=["dir"],
|
||||||
|
|||||||
@@ -754,10 +754,7 @@ def test_add_file_in_symlink_dir(make_tmp_dir, tmp_dir, dvc):
|
|||||||
def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys):
|
def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys):
|
||||||
tmp_dir.gen("foo", "foo")
|
tmp_dir.gen("foo", "foo")
|
||||||
|
|
||||||
mocker.patch(
|
mocker.patch("dvc_data.hashfile.checkout.test_links", return_value=[])
|
||||||
"dvc_data.hashfile.checkout.test_links",
|
|
||||||
return_value=[],
|
|
||||||
)
|
|
||||||
dvc.add("foo")
|
dvc.add("foo")
|
||||||
err = capsys.readouterr()[1]
|
err = capsys.readouterr()[1]
|
||||||
assert "reconfigure cache types" in err
|
assert "reconfigure cache types" in err
|
||||||
@@ -981,10 +978,7 @@ def test_add_does_not_remove_stage_file_on_failure(tmp_dir, dvc, mocker, target)
|
|||||||
dvcfile_contents = (tmp_dir / stage.path).read_text()
|
dvcfile_contents = (tmp_dir / stage.path).read_text()
|
||||||
|
|
||||||
exc_msg = f"raising error from mocked '{target}'"
|
exc_msg = f"raising error from mocked '{target}'"
|
||||||
mocker.patch(
|
mocker.patch(target, side_effect=DvcException(exc_msg))
|
||||||
target,
|
|
||||||
side_effect=DvcException(exc_msg),
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(DvcException, match=exc_msg):
|
with pytest.raises(DvcException, match=exc_msg):
|
||||||
dvc.add("foo")
|
dvc.add("foo")
|
||||||
|
|||||||
@@ -186,12 +186,7 @@ def test_checkout_empty_dir(tmp_dir, dvc):
|
|||||||
|
|
||||||
def test_checkout_not_cached_file(tmp_dir, dvc):
|
def test_checkout_not_cached_file(tmp_dir, dvc):
|
||||||
tmp_dir.dvc_gen("foo", "foo")
|
tmp_dir.dvc_gen("foo", "foo")
|
||||||
dvc.run(
|
dvc.run(cmd="cp foo bar", deps=["foo"], outs_no_cache=["bar"], name="copy-file")
|
||||||
cmd="cp foo bar",
|
|
||||||
deps=["foo"],
|
|
||||||
outs_no_cache=["bar"],
|
|
||||||
name="copy-file",
|
|
||||||
)
|
|
||||||
stats = dvc.checkout(force=True)
|
stats = dvc.checkout(force=True)
|
||||||
assert not any(stats.values())
|
assert not any(stats.values())
|
||||||
|
|
||||||
@@ -520,12 +515,7 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link):
|
|||||||
|
|
||||||
def test_checkout_with_deps(tmp_dir, dvc):
|
def test_checkout_with_deps(tmp_dir, dvc):
|
||||||
tmp_dir.dvc_gen({"foo": "foo"})
|
tmp_dir.dvc_gen({"foo": "foo"})
|
||||||
dvc.run(
|
dvc.run(cmd="echo foo > bar", outs=["bar"], deps=["foo"], name="copy-file")
|
||||||
cmd="echo foo > bar",
|
|
||||||
outs=["bar"],
|
|
||||||
deps=["foo"],
|
|
||||||
name="copy-file",
|
|
||||||
)
|
|
||||||
|
|
||||||
(tmp_dir / "bar").unlink()
|
(tmp_dir / "bar").unlink()
|
||||||
(tmp_dir / "foo").unlink()
|
(tmp_dir / "foo").unlink()
|
||||||
|
|||||||
@@ -10,9 +10,7 @@ from dvc.cli import main
|
|||||||
from dvc.exceptions import CheckoutError
|
from dvc.exceptions import CheckoutError
|
||||||
from dvc.repo.open_repo import clean_repos
|
from dvc.repo.open_repo import clean_repos
|
||||||
from dvc.stage.exceptions import StageNotFound
|
from dvc.stage.exceptions import StageNotFound
|
||||||
from dvc.testing.remote_tests import ( # noqa: F401
|
from dvc.testing.remote_tests import TestRemote # noqa: F401
|
||||||
TestRemote,
|
|
||||||
)
|
|
||||||
from dvc.utils.fs import remove
|
from dvc.utils.fs import remove
|
||||||
from dvc_data.hashfile.db import HashFileDB
|
from dvc_data.hashfile.db import HashFileDB
|
||||||
from dvc_data.hashfile.db.local import LocalHashFileDB
|
from dvc_data.hashfile.db.local import LocalHashFileDB
|
||||||
@@ -177,10 +175,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
|
|||||||
assert bar in caplog.text
|
assert bar in caplog.text
|
||||||
|
|
||||||
caplog.clear()
|
caplog.clear()
|
||||||
assert dvc.status(cloud=True) == {
|
assert dvc.status(cloud=True) == {"bar": "missing", "foo": "missing"}
|
||||||
"bar": "missing",
|
|
||||||
"foo": "missing",
|
|
||||||
}
|
|
||||||
assert header not in caplog.text
|
assert header not in caplog.text
|
||||||
assert foo not in caplog.text
|
assert foo not in caplog.text
|
||||||
assert bar not in caplog.text
|
assert bar not in caplog.text
|
||||||
|
|||||||
@@ -84,10 +84,7 @@ def test_directory(M, tmp_dir, dvc, scm):
|
|||||||
assert dvc.data_status(granular=True, untracked_files="all") == {
|
assert dvc.data_status(granular=True, untracked_files="all") == {
|
||||||
**EMPTY_STATUS,
|
**EMPTY_STATUS,
|
||||||
"committed": {
|
"committed": {
|
||||||
"added": M.unordered(
|
"added": M.unordered(join("dir", "bar"), join("dir", "foobar")),
|
||||||
join("dir", "bar"),
|
|
||||||
join("dir", "foobar"),
|
|
||||||
),
|
|
||||||
"modified": [join("dir", "")],
|
"modified": [join("dir", "")],
|
||||||
},
|
},
|
||||||
"uncommitted": {
|
"uncommitted": {
|
||||||
@@ -137,10 +134,7 @@ def test_new_empty_git_repo(M, tmp_dir, scm):
|
|||||||
dvc = Repo.init()
|
dvc = Repo.init()
|
||||||
assert dvc.data_status() == {
|
assert dvc.data_status() == {
|
||||||
**EMPTY_STATUS,
|
**EMPTY_STATUS,
|
||||||
"git": M.dict(
|
"git": M.dict(is_empty=True, is_dirty=True),
|
||||||
is_empty=True,
|
|
||||||
is_dirty=True,
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -180,10 +174,7 @@ def test_outs_with_no_hashes(M, tmp_dir, dvc, scm):
|
|||||||
dvc.stage.add(single_stage=True, outs=["bar"])
|
dvc.stage.add(single_stage=True, outs=["bar"])
|
||||||
dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
|
dvc.stage.add(deps=["bar"], outs=["foo"], name="copy", cmd="cp foo bar")
|
||||||
|
|
||||||
expected_output = {
|
expected_output = {**EMPTY_STATUS, "git": M.dict()}
|
||||||
**EMPTY_STATUS,
|
|
||||||
"git": M.dict(),
|
|
||||||
}
|
|
||||||
assert dvc.data_status() == expected_output
|
assert dvc.data_status() == expected_output
|
||||||
assert dvc.data_status(granular=True) == expected_output
|
assert dvc.data_status(granular=True) == expected_output
|
||||||
|
|
||||||
@@ -251,10 +242,7 @@ def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
|
|||||||
"untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
|
"untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
|
||||||
"committed": {"added": M.unordered("foobar", join("dir", ""))},
|
"committed": {"added": M.unordered("foobar", join("dir", ""))},
|
||||||
"uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))},
|
"uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))},
|
||||||
"not_in_cache": M.unordered(
|
"not_in_cache": M.unordered("foobar", join("dir", "")),
|
||||||
"foobar",
|
|
||||||
join("dir", ""),
|
|
||||||
),
|
|
||||||
"git": M.dict(),
|
"git": M.dict(),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -297,10 +285,7 @@ def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm):
|
|||||||
}
|
}
|
||||||
assert dvc.data_status(granular=True) == {
|
assert dvc.data_status(granular=True) == {
|
||||||
**EMPTY_STATUS,
|
**EMPTY_STATUS,
|
||||||
"not_in_cache": M.unordered(
|
"not_in_cache": M.unordered("foobar", join("dir", "")),
|
||||||
"foobar",
|
|
||||||
join("dir", ""),
|
|
||||||
),
|
|
||||||
"uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))},
|
"uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))},
|
||||||
"git": M.dict(),
|
"git": M.dict(),
|
||||||
"unchanged": M.unordered("foobar", join("dir", "")),
|
"unchanged": M.unordered("foobar", join("dir", "")),
|
||||||
@@ -389,9 +374,7 @@ def test_missing_dir_object_from_index(M, tmp_dir, dvc, scm):
|
|||||||
}
|
}
|
||||||
assert dvc.data_status(granular=True) == {
|
assert dvc.data_status(granular=True) == {
|
||||||
**EMPTY_STATUS,
|
**EMPTY_STATUS,
|
||||||
"committed": {
|
"committed": {"modified": [join("dir", "")]},
|
||||||
"modified": [join("dir", "")],
|
|
||||||
},
|
|
||||||
"uncommitted": {"unknown": [join("dir", "foobar")]},
|
"uncommitted": {"unknown": [join("dir", "foobar")]},
|
||||||
"not_in_cache": [join("dir", "")],
|
"not_in_cache": [join("dir", "")],
|
||||||
"git": M.dict(),
|
"git": M.dict(),
|
||||||
|
|||||||
@@ -106,10 +106,7 @@ def test_load_all_multistage(tmp_dir, dvc):
|
|||||||
metrics=["bar2"],
|
metrics=["bar2"],
|
||||||
always_changed=True,
|
always_changed=True,
|
||||||
)
|
)
|
||||||
assert set(load_file(dvc, PROJECT_FILE).stages.values()) == {
|
assert set(load_file(dvc, PROJECT_FILE).stages.values()) == {stage2, stage1}
|
||||||
stage2,
|
|
||||||
stage1,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_load_all_singlestage(tmp_dir, dvc):
|
def test_load_all_singlestage(tmp_dir, dvc):
|
||||||
|
|||||||
@@ -93,10 +93,7 @@ def test_pull_subdir_file(tmp_dir, erepo_dir):
|
|||||||
|
|
||||||
dest = tmp_dir / "file"
|
dest = tmp_dir / "file"
|
||||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||||
repo.dvcfs.get(
|
repo.dvcfs.get("subdir/file", os.fspath(dest))
|
||||||
"subdir/file",
|
|
||||||
os.fspath(dest),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert dest.is_file()
|
assert dest.is_file()
|
||||||
assert dest.read_text() == "contents"
|
assert dest.read_text() == "contents"
|
||||||
@@ -191,10 +188,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
|
|||||||
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
||||||
|
|
||||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||||
repo.dvcfs.get(
|
repo.dvcfs.get("dir", os.fspath(tmp_dir / "out"))
|
||||||
"dir",
|
|
||||||
os.fspath(tmp_dir / "out"),
|
|
||||||
)
|
|
||||||
expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"}
|
expected_files = {"foo": "foo", "bar": "bar", ".gitignore": "/foo\n"}
|
||||||
assert (tmp_dir / "out").read_text() == expected_files
|
assert (tmp_dir / "out").read_text() == expected_files
|
||||||
|
|
||||||
@@ -246,9 +240,6 @@ def test_subrepos_are_ignored_for_git_tracked_dirs(tmp_dir, erepo_dir):
|
|||||||
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
subrepo.dvc_gen({"file": "file"}, commit="add files on subrepo")
|
||||||
|
|
||||||
with external_repo(os.fspath(erepo_dir)) as repo:
|
with external_repo(os.fspath(erepo_dir)) as repo:
|
||||||
repo.dvcfs.get(
|
repo.dvcfs.get("dir", os.fspath(tmp_dir / "out"))
|
||||||
"dir",
|
|
||||||
os.fspath(tmp_dir / "out"),
|
|
||||||
)
|
|
||||||
# subrepo files should not be here
|
# subrepo files should not be here
|
||||||
assert (tmp_dir / "out").read_text() == scm_files
|
assert (tmp_dir / "out").read_text() == scm_files
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user