Compare commits

...
11 Commits
Author SHA1 Message Date
Dave Berenbaum ef226a96cd analytics: fix mac not sending reports (#10026) 2023-10-16 23:28:29 +03:00
Heinz-Alexander Fuetterer 8ea9d3edef chore: fix typos (#10022) 2023-10-14 12:54:57 +05:45
dependabot[bot] cafcefa758 build(deps-dev): bump pylint from 2.17.6 to 3.0.1
Bumps [pylint](https://github.com/pylint-dev/pylint) from 2.17.6 to 3.0.1.
- [Release notes](https://github.com/pylint-dev/pylint/releases)
- [Commits](https://github.com/pylint-dev/pylint/compare/v2.17.6...v3.0.1)

---
updated-dependencies:
- dependency-name: pylint
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>
2023-10-13 23:50:16 +03:00
Ruslan Kuprieiev 72fd265a8f get-url/ls-url: support remote:// with in-repo config 2023-10-13 18:58:23 +03:00
Brady Johnson c2f8115c9a exp rename: verify exp name follows ref format 2023-10-13 11:27:05 +09:00
Peter Rowlands e5dd2bb2cc cache migrate: add --dvc-files flag to migrate .dvc and dvc.lock files 2023-10-12 09:47:08 +09:00
Ruslan Kuprieiev 50d6788411 exp: preserve hidden alias -M for --message (#10006)
Followup for #10005
2023-10-10 04:41:43 +03:00
Matt Seddon 37a4a36dc7 exp save: use -m for --message short option 2023-10-09 09:03:44 -04:00
Dave Berenbaum 279f8cb68f repro --pull: don't fail for cloud-versioned remotes (#10001)
* repro --pull: don't fail for cloud-versioned remotes

* raise RunCacheNotSupported for cloud-versioned remotes
2023-10-06 18:11:30 +03:00
dberenbaum 8a70877748 fix artifact name test 2023-10-05 13:39:06 -04:00
dberenbaum f60091e64f bump gto 2023-10-05 13:39:06 -04:00
31 changed files with 232 additions and 50 deletions
+11
View File
@@ -40,8 +40,11 @@ class CmdCacheDir(CmdConfig):
class CmdCacheMigrate(CmdBase):
def run(self):
from dvc.cachemgr import migrate_2_to_3
from dvc.repo.commit import commit_2_to_3
migrate_2_to_3(self.repo, dry=self.args.dry)
if self.args.dvc_files:
commit_2_to_3(self.repo, dry=self.args.dry)
return 0
@@ -102,6 +105,14 @@ def add_parser(subparsers, parent_parser):
help=CACHE_MIGRATE_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
cache_migrate_parser.add_argument(
"--dvc-files",
help=(
"Migrate entries in all existing DVC files in the repository "
"to the DVC 3.0 format."
),
action="store_true",
)
cache_migrate_parser.add_argument(
"--dry",
help=(
+1 -1
View File
@@ -49,7 +49,7 @@ def add_parser(experiments_subparsers, parent_parser):
),
)
exec_run_parser.add_argument(
"-M",
"-m",
"--message",
type=str,
default=None,
+5
View File
@@ -106,3 +106,8 @@ def _add_run_common(parser):
default=None,
help="Custom commit message to use when committing the experiment.",
)
parser.add_argument(
"-M", # obsolete
dest="message",
help=argparse.SUPPRESS,
)
+6 -1
View File
@@ -72,10 +72,15 @@ def add_parser(experiments_subparsers, parent_parser):
metavar="<path>",
)
save_parser.add_argument(
"-M",
"-m",
"--message",
type=str,
default=None,
help="Custom commit message to use when committing the experiment.",
)
save_parser.add_argument(
"-M", # obsolete
dest="message",
help=argparse.SUPPRESS,
)
save_parser.set_defaults(func=CmdExperimentsSave)
+2
View File
@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
class CmdGetUrl(CmdBaseNoRepo):
def run(self):
from dvc.config import Config
from dvc.repo import Repo
try:
@@ -20,6 +21,7 @@ class CmdGetUrl(CmdBaseNoRepo):
jobs=self.args.jobs,
force=self.args.force,
fs_config=self.args.fs_config,
config=Config.from_cwd(),
)
return 0
except DvcException:
+2
View File
@@ -11,12 +11,14 @@ logger = logging.getLogger(__name__)
class CmdListUrl(CmdBaseNoRepo):
def run(self):
from dvc.config import Config
from dvc.repo import Repo
entries = Repo.ls_url(
self.args.url,
recursive=self.args.recursive,
fs_config=self.args.fs_config,
config=Config.from_cwd(),
)
if entries:
show_entries(entries, with_color=True, with_size=self.args.size)
+2 -8
View File
@@ -3,7 +3,6 @@
import inspect
import logging
import os
import platform
import sys
from subprocess import Popen # nosec B404
from typing import List
@@ -93,13 +92,8 @@ def _spawn_posix(cmd, env):
sys.stderr.close()
os.closerange(0, 3)
if platform.system() == "Darwin":
# workaround for MacOS bug
# https://github.com/iterative/dvc/issues/4294
_popen(cmd, env=env).communicate()
else:
os.environ.update(env)
main(cmd)
os.environ.update(env)
main(cmd)
os._exit(0) # pylint: disable=protected-access
+1 -1
View File
@@ -135,7 +135,7 @@ class DataCloud:
remote = self.get_remote(name=name, command=command)
if remote.fs.version_aware or remote.worktree:
raise NoRemoteError(
raise RemoteConfigError(
f"'{command}' is unsupported for cloud versioned remotes"
)
if hash_name in LEGACY_HASH_NAMES:
+4 -4
View File
@@ -88,11 +88,11 @@ def download(
return len(to_infos)
def parse_external_url(url, config=None):
remote_config = dict(config) if config else {}
def parse_external_url(url, fs_config=None, config=None):
remote_config = dict(fs_config) if fs_config else {}
remote_config["url"] = url
fs_cls, fs_config, fs_path = get_cloud_fs({}, **remote_config)
fs = fs_cls(**fs_config)
fs_cls, resolved_fs_config, fs_path = get_cloud_fs(config, **remote_config)
fs = fs_cls(**resolved_fs_config)
return fs, fs_path
+4 -4
View File
@@ -721,7 +721,7 @@ class Output:
if self.metric:
self.verify_metric()
self._update_legacy_hash_name()
self.update_legacy_hash_name()
if self.use_cache:
_, self.meta, self.obj = self._build(
self.cache,
@@ -745,8 +745,8 @@ class Output:
self.hash_info = self.obj.hash_info
self.files = None
def _update_legacy_hash_name(self):
if self.hash_name == "md5-dos2unix" and self.changed_checksum():
def update_legacy_hash_name(self, force: bool = False):
if self.hash_name == "md5-dos2unix" and (force or self.changed_checksum()):
self.hash_name = "md5"
def set_exec(self) -> None:
@@ -1391,7 +1391,7 @@ class Output:
)
assert self.repo
self._update_legacy_hash_name()
self.update_legacy_hash_name()
cache = self.cache if self.use_cache else self.local_cache
assert isinstance(cache, HashFileDB)
+51
View File
@@ -1,6 +1,12 @@
from typing import TYPE_CHECKING
from dvc import prompt
from . import locked
from .scm_context import scm_context
if TYPE_CHECKING:
from . import Repo
def _prepare_message(stage, changes):
@@ -68,3 +74,48 @@ def commit(
)
stage.dump(update_pipeline=False)
return [s.stage for s in stages_info]
@locked
@scm_context
def commit_2_to_3(repo: "Repo", dry: bool = False):
"""Force-commit all legacy outputs to use DVC 3.0 hashes."""
from dvc.dvcfile import ProjectFile
from dvc.ui import ui
view = repo.index.targets_view(
targets=None,
outs_filter=lambda o: o.hash_name == "md5-dos2unix",
recursive=True,
)
migrated = set()
for out in view.outs:
dvcfile = out.stage.dvcfile.relpath
if isinstance(out.stage.dvcfile, ProjectFile):
# pylint: disable-next=protected-access
lockfile = out.stage.dvcfile._lockfile.relpath
migrated.add(f"{dvcfile} ({lockfile})")
else:
migrated.add(dvcfile)
if not migrated:
ui.write("No DVC files in the repo to migrate to the 3.0 format.")
return
if dry:
ui.write("Entries in following DVC files will be migrated to the 3.0 format:")
ui.write("\n".join(sorted(f"\t{file}" for file in migrated)))
return
for stage, filter_info in view._stage_infos: # pylint: disable=protected-access
outs_filter = view._outs_filter # pylint: disable=protected-access
outs = {
out
for out in stage.filter_outs(filter_info)
if outs_filter is not None and outs_filter(out)
}
if outs:
for out in outs:
out.update_legacy_hash_name(force=True)
stage.save(allow_missing=True)
stage.commit(allow_missing=True, relink=True)
if not isinstance(stage.dvcfile, ProjectFile):
ui.write(f"Updating DVC file '{stage.dvcfile.relpath}'")
stage.dump(update_pipeline=False)
+1 -1
View File
@@ -241,7 +241,7 @@ class Experiments:
self.check_baseline(last_applied)
return last_applied
except BaselineMismatchError:
# If HEAD has moved since the the last applied experiment,
# If HEAD has moved since the last applied experiment,
# the applied experiment is no longer relevant
self.scm.remove_ref(EXEC_APPLY)
return None
+2 -1
View File
@@ -5,7 +5,7 @@ from dvc.repo.experiments.exceptions import (
ExperimentExistsError,
UnresolvedExpNamesError,
)
from dvc.repo.experiments.utils import resolve_name
from dvc.repo.experiments.utils import check_ref_format, resolve_name
from dvc.scm import Git
from .refs import ExpRefInfo
@@ -43,6 +43,7 @@ def rename(
if repo.scm.get_ref(str(new_ref)) and not force:
raise ExperimentExistsError(new_name)
check_ref_format(repo.scm, new_ref)
_rename_exp(scm=repo.scm, ref_info=result, new_name=new_name)
renamed.append(name)
+2 -2
View File
@@ -6,12 +6,12 @@ from dvc.fs import download, parse_external_url
from dvc.utils import resolve_output
def get_url(url, out=None, *, fs_config=None, jobs=None, force=False):
def get_url(url, out=None, *, fs_config=None, jobs=None, force=False, config=None):
out = resolve_output(url, out, force=force)
out = os.path.abspath(out)
(out,) = output.loads_from(None, [out], use_cache=False)
src_fs, src_path = parse_external_url(url, fs_config)
src_fs, src_path = parse_external_url(url, fs_config, config=config)
if not src_fs.exists(src_path):
raise URLMissingError(url)
download(src_fs, src_path, out.fs_path, jobs=jobs)
+2 -2
View File
@@ -2,8 +2,8 @@ from dvc.exceptions import URLMissingError
from dvc.fs import parse_external_url
def ls_url(url, *, fs_config=None, recursive=False):
fs, fs_path = parse_external_url(url, config=fs_config)
def ls_url(url, *, fs_config=None, recursive=False, config=None):
fs, fs_path = parse_external_url(url, fs_config=fs_config, config=config)
try:
info = fs.info(fs_path)
except FileNotFoundError as exc:
+1 -1
View File
@@ -261,7 +261,7 @@ class StageLoad:
Args:
path: if not provided, default `dvc.yaml` is assumed.
name: required for `dvc.yaml` files, ignored for `.dvc` files.
accept_group: if true, all of the the stages generated from `name`
accept_group: if true, all of the stages generated from `name`
foreach are returned.
glob: if true, `name` is considered as a glob, which is
used to filter list of stages from the given `path`.
+13 -6
View File
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
from funcy import first
from dvc import fs
from dvc.config import RemoteConfigError
from dvc.exceptions import CollectCacheError, DvcException
from dvc.utils import dict_sha256, relpath
@@ -274,15 +275,21 @@ class StageCache:
return ret
def push(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
dest_odb = odb or self.repo.cloud.get_remote_odb(
remote, "push --run-cache", hash_name="md5-dos2unix"
)
try:
dest_odb = odb or self.repo.cloud.get_remote_odb(
remote, "push --run-cache", hash_name="md5-dos2unix"
)
except RemoteConfigError as e:
raise RunCacheNotSupported(e) from e
return self.transfer(self.repo.cache.legacy, dest_odb)
def pull(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
odb = odb or self.repo.cloud.get_remote_odb(
remote, "fetch --run-cache", hash_name="md5-dos2unix"
)
try:
odb = odb or self.repo.cloud.get_remote_odb(
remote, "fetch --run-cache", hash_name="md5-dos2unix"
)
except RemoteConfigError as e:
raise RunCacheNotSupported(e) from e
return self.transfer(odb, self.repo.cache.legacy)
def get_used_objs(self, used_run_cache, *args, **kwargs):
+1 -1
View File
@@ -205,7 +205,7 @@ def boxify(message, border_color=None):
def _visual_width(line):
"""Get the the number of columns required to display a string"""
"""Get the number of columns required to display a string"""
return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line))
+2 -2
View File
@@ -44,7 +44,7 @@ dependencies = [
"flufl.lock>=5,<8",
"funcy>=1.14",
"grandalf<1,>=0.7",
"gto>=1.3.0,<2",
"gto>=1.4.0,<2",
"hydra-core>=1.1",
"iterative-telemetry>=0.0.7",
"networkx>=2.5",
@@ -77,7 +77,7 @@ gs = ["dvc-gs==2.22.1"]
hdfs = ["dvc-hdfs==2.19"]
lint = [
"mypy==1.5.1",
"pylint==2.17.6",
"pylint==3.0.1",
"types-colorama",
"types-psutil",
"types-pyinstaller",
+2 -2
View File
@@ -43,7 +43,7 @@ def test_artifacts_read_subdir(tmp_dir, dvc):
def test_artifacts_read_bad_name(tmp_dir, dvc, caplog):
bad_name_dvcyaml = deepcopy(dvcyaml)
bad_name_dvcyaml["artifacts"]["Bad_name"] = {"type": "model", "path": "bad.pkl"}
bad_name_dvcyaml["artifacts"]["_bad_name_"] = {"type": "model", "path": "bad.pkl"}
(tmp_dir / "dvc.yaml").dump(bad_name_dvcyaml)
@@ -54,7 +54,7 @@ def test_artifacts_read_bad_name(tmp_dir, dvc, caplog):
with caplog.at_level(logging.WARNING):
assert tmp_dir.dvc.artifacts.read() == {"dvc.yaml": artifacts}
assert "Can't use 'Bad_name' as artifact name (ID)" in caplog.text
assert "Can't use '_bad_name_' as artifact name (ID)" in caplog.text
def test_artifacts_add_subdir(tmp_dir, dvc):
+11
View File
@@ -1,5 +1,6 @@
import pytest
from dvc.exceptions import InvalidArgumentError
from dvc.repo.experiments.exceptions import (
ExperimentExistsError,
UnresolvedExpNamesError,
@@ -31,3 +32,13 @@ def test_existing_name(dvc, exp_stage):
dvc.experiments.rename("second-name", "first-name")
dvc.experiments.rename("second-name", "first-name", force=True)
def test_invalid_name(dvc, exp_stage):
dvc.experiments.run(exp_stage.addressing, name="test-name", params=["foo=1"])
with pytest.raises(InvalidArgumentError):
dvc.experiments.rename("invalid*name", "test-name")
with pytest.raises(InvalidArgumentError):
dvc.experiments.rename("invalid/name", "test-name")
+2 -2
View File
@@ -442,7 +442,7 @@ def test_repro_list_of_commands_raise_and_stops_after_failure(tmp_dir, dvc, mult
assert not (tmp_dir / "bar").exists()
def test_repro_pulls_mising_data_source(tmp_dir, dvc, mocker, local_remote):
def test_repro_pulls_missing_data_source(tmp_dir, dvc, mocker, local_remote):
(foo,) = tmp_dir.dvc_gen("foo", "foo")
dvc.push()
@@ -454,7 +454,7 @@ def test_repro_pulls_mising_data_source(tmp_dir, dvc, mocker, local_remote):
assert dvc.reproduce(pull=True)
def test_repro_pulls_mising_import(tmp_dir, dvc, mocker, erepo_dir, local_remote):
def test_repro_pulls_missing_import(tmp_dir, dvc, mocker, erepo_dir, local_remote):
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "foo", commit="first")
+2 -2
View File
@@ -4,7 +4,7 @@ from dvc.stage.cache import RunCacheNotSupported
from dvc.utils.fs import remove
def test_repro_pulls_mising_data_source(tmp_dir, dvc, mocker, local_remote):
def test_repro_pulls_missing_data_source(tmp_dir, dvc, mocker, local_remote):
(foo,) = tmp_dir.dvc_gen("foo", "foo")
dvc.push()
@@ -16,7 +16,7 @@ def test_repro_pulls_mising_data_source(tmp_dir, dvc, mocker, local_remote):
assert dvc.reproduce(pull=True)
def test_repro_pulls_mising_import(tmp_dir, dvc, mocker, erepo_dir, local_remote):
def test_repro_pulls_missing_import(tmp_dir, dvc, mocker, erepo_dir, local_remote):
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "foo", commit="first")
+10
View File
@@ -3,6 +3,7 @@ import os
import pytest
from dvc.config import Config
from dvc.exceptions import FileExistsLocallyError, URLMissingError
from dvc.repo import Repo
from dvc.testing.workspace_tests import TestGetUrl as _TestGetUrl
@@ -66,3 +67,12 @@ def test_get_url_nonexistent(tmp_dir):
class TestGetUrl(_TestGetUrl):
pass
def test_get_url_config(tmp_dir, dvc, make_remote):
remote_path = make_remote("myremote", default=False, typ="local")
(remote_path / "foo").write_text("foo")
(remote_path / "bar").write_text("bar")
Repo.get_url("remote://myremote/foo", config=Config.from_cwd())
assert (tmp_dir / "foo").read_text() == "foo"
+18
View File
@@ -1,5 +1,23 @@
from dvc.config import Config
from dvc.repo import Repo
from dvc.testing.workspace_tests import TestLsUrl as _TestLsUrl
class TestLsUrl(_TestLsUrl):
pass
def test_ls_url_config(dvc, make_remote):
remote_path = make_remote("myremote", default=False, typ="local")
(remote_path / "foo").write_text("foo")
(remote_path / "bar").write_text("bar")
actual = sorted(
Repo.ls_url("remote://myremote", config=Config.from_cwd()),
key=lambda entry: entry["path"],
)
expected = [
{"isdir": False, "path": "bar", "size": 3},
{"isdir": False, "path": "foo", "size": 3},
]
assert actual == expected
+13 -1
View File
@@ -4,7 +4,7 @@ import pytest
from funcy import first
from dvc.dvcfile import LOCK_FILE
from dvc.stage.cache import _get_stage_hash
from dvc.stage.cache import RunCacheNotSupported, _get_stage_hash
from dvc.utils.fs import remove
@@ -229,3 +229,15 @@ def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote):
assert (tmp_dir / "bar").exists()
assert not (tmp_dir / "foo").unlink()
assert (tmp_dir / LOCK_FILE).exists()
def test_push_pull_unsupported(tmp_dir, dvc, mocker, run_copy):
tmp_dir.gen("foo", "foo")
run_copy("foo", "bar", name="copy-foo-bar")
mocker.patch.object(
dvc.cloud, "get_remote_odb", side_effect=RunCacheNotSupported("foo")
)
with pytest.raises(RunCacheNotSupported):
dvc.push(run_cache=True)
with pytest.raises(RunCacheNotSupported):
dvc.pull(run_cache=True)
+40
View File
@@ -150,6 +150,27 @@ def test_experiments_run(dvc, scm, mocker):
cmd.repo.experiments.run.assert_called_with(**default_arguments)
@pytest.mark.parametrize("flag", ["-m", "-M", "--message"])
def test_experiments_run_message(dvc, scm, mocker, flag):
default_arguments = {
"params": [],
"name": None,
"queue": False,
"run_all": False,
"jobs": 1,
"tmp_dir": False,
"copy_paths": [],
"message": "mymessage",
}
default_arguments.update(repro_arguments)
cmd = CmdExperimentsRun(parse_args(["exp", "run", flag, "mymessage"]))
mocker.patch.object(cmd.repo, "reproduce")
mocker.patch.object(cmd.repo.experiments, "run")
cmd.run()
cmd.repo.experiments.run.assert_called_with(**default_arguments)
def test_experiments_branch(dvc, scm, mocker):
m = mocker.patch("dvc.repo.experiments.branch.branch", return_value={})
@@ -479,6 +500,25 @@ def test_experiments_save(dvc, scm, mocker):
)
@pytest.mark.parametrize("flag", ["-m", "-M", "--message"])
def test_experiments_save_message(dvc, scm, mocker, flag):
cli_args = parse_args(["exp", "save", flag, "custom commit message"])
assert cli_args.func == CmdExperimentsSave
cmd = cli_args.func(cli_args)
m = mocker.patch("dvc.repo.experiments.save.save", return_value="acabb")
assert cmd.run() == 0
m.assert_called_once_with(
cmd.repo,
name=None,
force=False,
include_untracked=[],
message="custom commit message",
)
def test_experiments_clean(dvc, scm, mocker):
cli_args = parse_args(["experiments", "clean"])
assert cli_args.func == CmdExperimentsClean
+10 -2
View File
@@ -1,8 +1,9 @@
from dvc.cli import parse_args
from dvc.commands.get_url import CmdGetUrl
from dvc.config import Config
def test_get_url(mocker):
def test_get_url(mocker, M):
cli_args = parse_args(["get-url", "src", "out", "-j", "5"])
assert cli_args.func == CmdGetUrl
@@ -11,4 +12,11 @@ def test_get_url(mocker):
assert cmd.run() == 0
m.assert_called_once_with("src", out="out", jobs=5, force=False, fs_config=None)
m.assert_called_once_with(
"src",
out="out",
jobs=5,
force=False,
fs_config=None,
config=M.instance_of(Config),
)
+9 -4
View File
@@ -1,8 +1,9 @@
from dvc.cli import parse_args
from dvc.commands.ls_url import CmdListUrl
from dvc.config import Config
def test_ls_url(mocker):
def test_ls_url(mocker, M):
cli_args = parse_args(["ls-url", "src"])
assert cli_args.func == CmdListUrl
cmd = cli_args.func(cli_args)
@@ -10,10 +11,12 @@ def test_ls_url(mocker):
assert cmd.run() == 0
m.assert_called_once_with("src", recursive=False, fs_config=None)
m.assert_called_once_with(
"src", recursive=False, fs_config=None, config=M.instance_of(Config)
)
def test_recursive(mocker):
def test_recursive(mocker, M):
cli_args = parse_args(["ls-url", "-R", "src"])
assert cli_args.func == CmdListUrl
cmd = cli_args.func(cli_args)
@@ -21,4 +24,6 @@ def test_recursive(mocker):
assert cmd.run() == 0
m.assert_called_once_with("src", recursive=True, fs_config=None)
m.assert_called_once_with(
"src", recursive=True, fs_config=None, config=M.instance_of(Config)
)
@@ -107,7 +107,7 @@ def test_workspace_executor_failed_status(dvc, scm, failed_exp_stage, queue_type
assert executor_info.status == TaskStatus.FAILED
def test_executor_status_compatiblity():
def test_executor_status_compatibility():
data = {
"git_url": "file:///Users/home",
"baseline_rev": "123",
+1 -1
View File
@@ -127,7 +127,7 @@ def test_branch_config(tmp_dir, scm):
dvc.close()
def test_dynamic_cache_initalization(tmp_dir, scm):
def test_dynamic_cache_initialization(tmp_dir, scm):
dvc = Repo.init()
with dvc.config.edit() as conf:
conf["cache"]["ssh"] = "foo"