diff --git a/dvc/command/experiments/push.py b/dvc/command/experiments/push.py index b596b5c0b..73b6becd9 100644 --- a/dvc/command/experiments/push.py +++ b/dvc/command/experiments/push.py @@ -88,6 +88,9 @@ def add_parser(experiments_subparsers, parent_parser): metavar="", ) experiments_push_parser.add_argument( - "experiment", help="Experiment to push.", metavar="" + "experiment", + nargs="+", + help="Experiments to push.", + metavar="", ).complete = completion.EXPERIMENT experiments_push_parser.set_defaults(func=CmdExperimentsPush) diff --git a/dvc/repo/experiments/exceptions.py b/dvc/repo/experiments/exceptions.py new file mode 100644 index 000000000..6befa3b65 --- /dev/null +++ b/dvc/repo/experiments/exceptions.py @@ -0,0 +1,41 @@ +from dvc.exceptions import InvalidArgumentError +from dvc.types import Iterable, List, Optional + +from .base import ExpRefInfo + + +class AmbiguousExpRefInfo(InvalidArgumentError): + def __init__( + self, + exp_name: str, + exp_ref_list: Iterable[ExpRefInfo], + ): + msg = [ + ( + f"Ambiguous name '{exp_name}' refers to multiple experiments." + " Use one of the following full refnames instead:" + ), + "", + ] + msg.extend([f"\t{info}" for info in exp_ref_list]) + super().__init__("\n".join(msg)) + +class UnresolvedExpNamesError(InvalidArgumentError): + def __init__( + self, unresolved_list: List[str], *args, git_remote: str = None + ): + unresolved_names = ";".join(unresolved_list) + if not git_remote: + if len(unresolved_names) > 1: + super().__init__( + f"'{unresolved_names}' are not valid experiment names" + ) + else: + super().__init__( + f"'{unresolved_names}' is not a valid experiment name" + ) + else: + super().__init__( + f"Experiment '{unresolved_names}' does not exist " + f"in '{git_remote}'" + ) diff --git a/dvc/repo/experiments/push.py b/dvc/repo/experiments/push.py index b0b43e482..700d86d08 100644 --- a/dvc/repo/experiments/push.py +++ b/dvc/repo/experiments/push.py @@ -1,9 +1,12 @@ import logging -from dvc.exceptions import DvcException, InvalidArgumentError +from dvc.exceptions import DvcException from dvc.repo import locked from dvc.repo.scm_context import scm_context +from dvc.scm import TqdmGit +from dvc.types import IterableStr, Union +from .exceptions import UnresolvedExpNamesError from .utils import exp_commits, push_refspec, resolve_name logger = logging.getLogger(__name__) @@ -13,50 +16,66 @@ logger = logging.getLogger(__name__) @scm_context def push( repo, - git_remote, - exp_name: str, + git_remote: str, + exp_names: Union[IterableStr, str], *args, - force=False, - push_cache=False, + force: bool = False, + push_cache: bool = False, **kwargs, ): - exp_ref_dict = resolve_name(repo.scm, exp_name) - exp_ref = exp_ref_dict[exp_name] - if not exp_ref: - raise InvalidArgumentError( - f"'{exp_name}' is not a valid experiment name" - ) + if isinstance(exp_names, str): + exp_names = [exp_names] + exp_ref_dict = resolve_name(repo.scm, exp_names) + unresolved_exp_names = [ + exp_name + for exp_name, exp_ref in exp_ref_dict.items() + if exp_ref is None + ] + if unresolved_exp_names: + raise UnresolvedExpNamesError(unresolved_exp_names) + + exp_ref_set = exp_ref_dict.values() + _push(repo, git_remote, exp_ref_set, force, push_cache, **kwargs) + + +def _push( + repo, + git_remote: str, + exp_refs, + force: bool, + push_cache: bool, + **kwargs, +): def on_diverged(refname: str, rev: str) -> bool: if repo.scm.get_ref(refname) == rev: return True + exp_name = refname.split("/")[-1] raise DvcException( f"Local experiment '{exp_name}' has diverged from remote " "experiment with the same name. To override the remote experiment " "re-run with '--force'." ) - refname = str(exp_ref) - logger.debug("git push experiment '%s' -> '%s'", exp_ref, git_remote) + logger.debug(f"git push experiment '{exp_refs}' -> '{git_remote}'") - from dvc.scm import TqdmGit - - with TqdmGit(desc="Pushing git refs") as pbar: - push_refspec( - repo.scm, - git_remote, - refname, - refname, - force=force, - on_diverged=on_diverged, - progress=pbar.update_git, - ) + for exp_ref in exp_refs: + with TqdmGit(desc="Pushing git refs") as pbar: + push_refspec( + repo.scm, + git_remote, + str(exp_ref), + str(exp_ref), + force=force, + on_diverged=on_diverged, + progress=pbar.update_git, + ) if push_cache: - _push_cache(repo, exp_ref, **kwargs) + _push_cache(repo, exp_refs, **kwargs) -def _push_cache(repo, exp_ref, dvc_remote=None, jobs=None, run_cache=False): - revs = list(exp_commits(repo.scm, [exp_ref])) - logger.debug("dvc push experiment '%s'", exp_ref) +def _push_cache(repo, exp_refs, dvc_remote=None, jobs=None, run_cache=False): + revs = list(exp_commits(repo.scm, exp_refs)) + logger.debug(f"dvc push experiment '{exp_refs}'") repo.push(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs) diff --git a/dvc/repo/experiments/utils.py b/dvc/repo/experiments/utils.py index c645c3224..239189f37 100644 --- a/dvc/repo/experiments/utils.py +++ b/dvc/repo/experiments/utils.py @@ -21,24 +21,9 @@ from .base import ( EXPS_STASH, ExpRefInfo, ) +from dvc.repo.experiments.exceptions import AmbiguousExpRefInfo -class AmbiguousExpRefInfo(InvalidArgumentError): - def __init__( - self, - exp_name: str, - exp_ref_list: Iterable[ExpRefInfo], - ): - msg = [ - ( - f"Ambiguous name '{exp_name}' refers to multiple experiments." - " Use one of the following full refnames instead:" - ), - "", - ] - msg.extend([f"\t{info}" for info in exp_ref_list]) - super().__init__("\n".join(msg)) - def exp_refs(scm: "Git") -> Generator["ExpRefInfo", None, None]: """Iterate over all experiment refs.""" diff --git a/dvc/types.py b/dvc/types.py index cca8e17a0..ee2449339 100644 --- a/dvc/types.py +++ b/dvc/types.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union if TYPE_CHECKING: from os import PathLike @@ -10,3 +10,4 @@ OptStr = Optional[str] TargetType = Union[List[str], str] DictStrAny = Dict[str, Any] DictAny = Dict[Any, Any] +IterableStr = Iterable[str] diff --git a/tests/func/experiments/test_remote.py b/tests/func/experiments/test_remote.py index 47bce63e8..e36230ac9 100644 --- a/tests/func/experiments/test_remote.py +++ b/tests/func/experiments/test_remote.py @@ -13,19 +13,30 @@ def test_push(tmp_dir, scm, dvc, git_upstream, exp_stage, use_url): remote = git_upstream.url if use_url else git_upstream.remote with pytest.raises(InvalidArgumentError): - dvc.experiments.push(remote, "foo") + dvc.experiments.push(remote, ["foo"]) + + results = dvc.experiments.run(exp_stage.addressing, params=["foo=1"]) + exp1 = first(results) + ref_info1 = first(exp_refs_by_rev(scm, exp1)) results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"]) - exp = first(results) - ref_info = first(exp_refs_by_rev(scm, exp)) + exp2 = first(results) + ref_info2 = first(exp_refs_by_rev(scm, exp2)) - dvc.experiments.push(remote, ref_info.name) - assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp + results = dvc.experiments.run(exp_stage.addressing, params=["foo=3"]) + exp3 = first(results) + ref_info3 = first(exp_refs_by_rev(scm, exp3)) - git_upstream.tmp_dir.scm.remove_ref(str(ref_info)) + dvc.experiments.push(remote, [ref_info1.name, ref_info2.name]) + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info1)) == exp1 + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info2)) == exp2 + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info3)) is None - dvc.experiments.push(remote, str(ref_info)) - assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp + git_upstream.tmp_dir.scm.remove_ref(str(ref_info1)) + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info1)) is None + + dvc.experiments.push(remote, [ref_info1.name]) + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info1)) == exp1 def test_push_diverged(tmp_dir, scm, dvc, git_upstream, exp_stage): @@ -39,10 +50,10 @@ def test_push_diverged(tmp_dir, scm, dvc, git_upstream, exp_stage): git_upstream.tmp_dir.scm.set_ref(str(ref_info), remote_rev) with pytest.raises(DvcException): - dvc.experiments.push(git_upstream.remote, ref_info.name) + dvc.experiments.push(git_upstream.remote, [ref_info.name]) assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == remote_rev - dvc.experiments.push(git_upstream.remote, ref_info.name, force=True) + dvc.experiments.push(git_upstream.remote, [ref_info.name], force=True) assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp @@ -53,7 +64,7 @@ def test_push_checkpoint(tmp_dir, scm, dvc, git_upstream, checkpoint_stage): exp_a = first(results) ref_info_a = first(exp_refs_by_rev(scm, exp_a)) - dvc.experiments.push(git_upstream.remote, ref_info_a.name, force=True) + dvc.experiments.push(git_upstream.remote, [ref_info_a.name], force=True) assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_a)) == exp_a results = dvc.experiments.run( @@ -64,7 +75,7 @@ def test_push_checkpoint(tmp_dir, scm, dvc, git_upstream, checkpoint_stage): tmp_dir.scm_gen("new", "new", commit="new") - dvc.experiments.push(git_upstream.remote, ref_info_b.name, force=True) + dvc.experiments.push(git_upstream.remote, [ref_info_b.name], force=True) assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_b)) == exp_b @@ -86,15 +97,15 @@ def test_push_ambiguous_name(tmp_dir, scm, dvc, git_upstream, exp_stage): exp_b = first(results) ref_info_b = first(exp_refs_by_rev(scm, exp_b)) - dvc.experiments.push(remote, "foo") + dvc.experiments.push(remote, ["foo"]) assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_b)) == exp_b tmp_dir.scm_gen("new", "new 2", commit="new 2") with pytest.raises(InvalidArgumentError): - dvc.experiments.push(remote, "foo") + dvc.experiments.push(remote, ["foo"]) - dvc.experiments.push(remote, str(ref_info_a)) + dvc.experiments.push(remote, [str(ref_info_a)]) assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_a)) == exp_a @@ -237,7 +248,7 @@ def test_push_pull_cache( exp = first(results) ref_info = first(exp_refs_by_rev(scm, exp)) - dvc.experiments.push(remote, ref_info.name, push_cache=True) + dvc.experiments.push(remote, [ref_info.name], push_cache=True) for x in range(2, checkpoint_stage.iterations + 1): hash_ = digest(str(x)) path = os.path.join(local_remote.url, hash_[:2], hash_[2:]) @@ -285,4 +296,4 @@ def test_auth_error_push(tmp_dir, scm, dvc, exp_stage, http_auth_patch): GitAuthError, match=f"HTTP Git authentication is not supported: '{http_auth_patch}'", ): - dvc.experiments.push(http_auth_patch, ref_info.name) + dvc.experiments.push(http_auth_patch, [ref_info.name]) diff --git a/tests/func/experiments/test_remove.py b/tests/func/experiments/test_remove.py index 30fd7d9e9..d51cbcca4 100644 --- a/tests/func/experiments/test_remove.py +++ b/tests/func/experiments/test_remove.py @@ -106,7 +106,7 @@ def test_remove_remote(tmp_dir, scm, dvc, exp_stage, git_upstream, use_url): exp_list.append(exp) ref_info = first(exp_refs_by_rev(scm, exp)) ref_info_list.append(ref_info) - dvc.experiments.push(remote, ref_info.name) + dvc.experiments.push(remote, [ref_info.name]) assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp dvc.experiments.remove( diff --git a/tests/unit/command/test_experiments.py b/tests/unit/command/test_experiments.py index dbc21c50c..0802e8221 100644 --- a/tests/unit/command/test_experiments.py +++ b/tests/unit/command/test_experiments.py @@ -221,7 +221,8 @@ def test_experiments_push(dvc, scm, mocker): "experiments", "push", "origin", - "experiment", + "experiment1", + "experiment2", "--force", "--no-cache", "--remote", @@ -241,7 +242,7 @@ def test_experiments_push(dvc, scm, mocker): m.assert_called_once_with( cmd.repo, "origin", - "experiment", + ["experiment1", "experiment2"], force=True, push_cache=False, dvc_remote="my-remote",