dvc: bump to 2.6.4

Fixes https://github.com/iterative/dvc/issues/6466

.codecov.yml

@@ -3,7 +3,7 @@ coverage:
   status:
     project:
       default:
-        target: 85
+        target: 90
         threshold: 10%
     patch:
       default:

dvc/analytics.py

@@ -2,7 +2,7 @@ import json
 import logging
 import os
 
-from .env import DVC_ANALYTICS
+from .env import DVC_NO_ANALYTICS
 
 logger = logging.getLogger(__name__)
 
@@ -44,8 +44,8 @@ def is_enabled():
     if env2bool("DVC_TEST"):
         return False
 
-    enabled = env2bool(DVC_ANALYTICS, None)
-    if enabled is None:
+    enabled = not os.getenv(DVC_NO_ANALYTICS)
+    if enabled:
         enabled = to_bool(
             Config(validate=False).get("core", {}).get("analytics", "true")
         )

dvc/command/experiments.py

@@ -747,7 +747,9 @@ class CmdExperimentsRemove(CmdBase):
     def run(self):
 
         self.repo.experiments.remove(
-            exp_names=self.args.experiment, queue=self.args.queue
+            exp_names=self.args.experiment,
+            queue=self.args.queue,
+            clear_all=self.args.all,
         )
 
         return 0
@@ -1237,9 +1239,16 @@ def add_parser(subparsers, parent_parser):
         help=EXPERIMENTS_REMOVE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    experiments_remove_parser.add_argument(
+    remove_group = experiments_remove_parser.add_mutually_exclusive_group()
+    remove_group.add_argument(
         "--queue", action="store_true", help="Remove all queued experiments."
     )
+    remove_group.add_argument(
+        "-A",
+        "--all",
+        action="store_true",
+        help="Remove all committed experiments.",
+    )
     experiments_remove_parser.add_argument(
         "experiment",
         nargs="*",

dvc/env.py

@@ -9,4 +9,4 @@ DVCLIVE_RESUME = "DVCLIVE_RESUME"
 DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY"
 DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE"
 DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH"
-DVC_ANALYTICS = "DVC_ANALYTICS"
+DVC_NO_ANALYTICS = "DVC_NO_ANALYTICS"

dvc/repo/experiments/__init__.py

@@ -455,7 +455,7 @@ class Experiments:
             )
         logger.info(
             "\nTo promote an experiment to a Git branch run:\n\n"
-            "\tdvc exp branch <exp>\n"
+            "\tdvc exp branch <exp> <branch>\n"
         )
 
     @scm_locked

dvc/repo/experiments/executor/base.py

@@ -348,6 +348,7 @@ class BaseExecutor(ABC):
                     quiet=True,
                     allow_missing=True,
                     checkpoint_reset=checkpoint_reset,
+                    recursive=kwargs.get("recursive", False),
                 )
 
             checkpoint_func = partial(

dvc/repo/experiments/remove.py

@@ -7,21 +7,29 @@ from dvc.repo.scm_context import scm_context
 from dvc.scm.base import RevError
 
 from .base import EXPS_NAMESPACE, ExpRefInfo
-from .utils import exp_refs_by_name, remove_exp_refs
+from .utils import exp_refs, exp_refs_by_name, remove_exp_refs
 
 logger = logging.getLogger(__name__)
 
 
 @locked
 @scm_context
-def remove(repo, exp_names=None, queue=False, **kwargs):
-    if not exp_names and not queue:
+def remove(
+    repo,
+    exp_names=None,
+    queue=False,
+    clear_all=False,
+    **kwargs,
+):
+    if not any([exp_names, queue, clear_all]):
         return 0
 
     removed = 0
     if queue:
-        removed += len(repo.experiments.stash)
-        repo.experiments.stash.clear()
+        removed += _clear_stash(repo)
+    if clear_all:
+        removed += _clear_all(repo)
+
     if exp_names:
         remained = _remove_commited_exps(repo, exp_names)
         remained = _remove_queued_exps(repo, remained)
@@ -33,6 +41,18 @@ def remove(repo, exp_names=None, queue=False, **kwargs):
     return removed
 
 
+def _clear_stash(repo):
+    removed = len(repo.experiments.stash)
+    repo.experiments.stash.clear()
+    return removed
+
+
+def _clear_all(repo):
+    ref_infos = list(exp_refs(repo.scm))
+    remove_exp_refs(repo.scm, ref_infos)
+    return len(ref_infos)
+
+
 def _get_exp_stash_index(repo, ref_or_rev: str) -> Optional[int]:
     stash_revs = repo.experiments.stash_revs
     for _, ref_info in stash_revs.items():
@@ -53,9 +73,9 @@ def _get_exp_ref(repo, exp_name: str) -> Optional[ExpRefInfo]:
         if repo.scm.get_ref(exp_name):
             return ExpRefInfo.from_ref(exp_name)
     else:
-        exp_refs = list(exp_refs_by_name(repo.scm, exp_name))
-        if exp_refs:
-            return _get_ref(exp_refs, exp_name, cur_rev)
+        exp_ref_list = list(exp_refs_by_name(repo.scm, exp_name))
+        if exp_ref_list:
+            return _get_ref(exp_ref_list, exp_name, cur_rev)
     return None
 
 

dvc/utils/serialize/_yaml.py

@@ -23,17 +23,10 @@ def load_yaml(path, fs=None):
 def parse_yaml(text, path, typ="safe"):
     from ruamel.yaml import YAML
     from ruamel.yaml import YAMLError as _YAMLError
-    from ruamel.yaml.constructor import DuplicateKeyError
 
     yaml = YAML(typ=typ)
-    try:
-        with reraise(_YAMLError, YAMLFileCorruptedError(path)):
-            return yaml.load(text) or {}
-    except DuplicateKeyError as exc:
-        # NOTE: unfortunately this one doesn't inherit from YAMLError, so we
-        # have to catch it by-hand. See
-        # https://yaml.readthedocs.io/en/latest/api.html#duplicate-keys
-        raise YAMLError(path, exc.problem)
+    with reraise(_YAMLError, YAMLFileCorruptedError(path)):
+        return yaml.load(text) or {}
 
 
 def parse_yaml_for_update(text, path):

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.6.2"
+_BASE_VERSION = "2.6.4"
 
 
 def _generate_version(base_version):

setup.py

@@ -61,7 +61,7 @@ install_requires = [
     "grandalf==0.6",
     "distro>=1.3.0",
     "appdirs>=1.4.3",
-    "ruamel.yaml>=0.16.1",
+    "ruamel.yaml>=0.17.11",
     "toml>=0.10.1",
     "funcy>=1.14",
     "pathspec>=0.6.0,<0.9.0",
@@ -97,7 +97,7 @@ install_requires = [
 
 gs = ["gcsfs==2021.7.0"]
 gdrive = ["pydrive2[fsspec]>=1.9.1"]
-s3 = ["s3fs==2021.7.0", "aiobotocore[boto3]>1.0.1"]
+s3 = ["s3fs==2021.8.0", "aiobotocore[boto3]>1.0.1"]
 azure = ["adlfs==2021.7.1", "azure-identity>=1.4.0", "knack"]
 oss = ["ossfs==2021.7.5"]
 ssh = ["sshfs>=2021.7.1"]

tests/func/experiments/test_experiments.py

@@ -8,7 +8,7 @@ from funcy import first
 
 from dvc.dvcfile import PIPELINE_FILE
 from dvc.repo.experiments.utils import exp_refs_by_rev
-from dvc.utils.serialize import PythonFileCorruptedError
+from dvc.utils.serialize import PythonFileCorruptedError, load_yaml
 from tests.func.test_repro_multistage import COPY_SCRIPT
 
 
@@ -657,3 +657,12 @@ def test_modified_data_dep(tmp_dir, scm, dvc, workspace, params, target):
     if workspace:
         assert (tmp_dir / "metrics.yaml").read_text().strip() == params
         assert (tmp_dir / "data").read_text().strip() == "modified"
+
+
+def test_exp_run_recursive(tmp_dir, scm, dvc, run_copy_metrics):
+    tmp_dir.dvc_gen("metric_t.json", "foo: 1")
+    run_copy_metrics(
+        "metric_t.json", "metric.json", metrics=["metric.json"], no_exec=True
+    )
+    assert dvc.experiments.run(".", recursive=True)
+    assert load_yaml(tmp_dir / "metric.json") == {"foo": 1}

tests/func/experiments/test_remove.py

@@ -72,3 +72,21 @@ def test_remove_special_queued_experiments(tmp_dir, scm, dvc, exp_stage):
     assert rev3 in dvc.experiments.stash_revs
     assert scm.get_ref(str(ref_info1)) is None
     assert scm.get_ref(str(ref_info2)) is not None
+
+
+def test_remove_all(tmp_dir, scm, dvc, exp_stage):
+    results = dvc.experiments.run(exp_stage.addressing, params=["foo=1"])
+    ref_info1 = first(exp_refs_by_rev(scm, first(results)))
+    dvc.experiments.run(exp_stage.addressing, params=["foo=2"], queue=True)
+    scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
+    scm.commit("update baseline")
+
+    results = dvc.experiments.run(exp_stage.addressing, params=["foo=3"])
+    ref_info2 = first(exp_refs_by_rev(scm, first(results)))
+    dvc.experiments.run(exp_stage.addressing, params=["foo=4"], queue=True)
+
+    removed = dvc.experiments.remove(clear_all=True)
+    assert removed == 2
+    assert len(dvc.experiments.stash) == 2
+    assert scm.get_ref(str(ref_info2)) is None
+    assert scm.get_ref(str(ref_info1)) is None

tests/unit/command/test_experiments.py

@@ -252,13 +252,25 @@ def test_experiments_pull(dvc, scm, mocker):
     )
 
 
-def test_experiments_remove(dvc, scm, mocker):
-    cli_args = parse_args(["experiments", "remove", "--queue"])
+@pytest.mark.parametrize(
+    "queue,clear_all",
+    [(True, False), (False, True)],
+)
+def test_experiments_remove(dvc, scm, mocker, queue, clear_all):
+    if queue:
+        args = "--queue"
+    if clear_all:
+        args = "--all"
+    cli_args = parse_args(["experiments", "remove", args])
     assert cli_args.func == CmdExperimentsRemove
 
     cmd = cli_args.func(cli_args)
     m = mocker.patch("dvc.repo.experiments.remove.remove", return_value={})
 
     assert cmd.run() == 0
-
-    m.assert_called_once_with(cmd.repo, exp_names=[], queue=True)
+    m.assert_called_once_with(
+        cmd.repo,
+        exp_names=[],
+        queue=queue,
+        clear_all=clear_all,
+    )

tests/unit/test_analytics.py

@@ -87,7 +87,7 @@ def test_is_enabled(dvc, config, result, monkeypatch, tmp_global_dir):
 
     # reset DVC_TEST env var, which affects `is_enabled()`
     monkeypatch.delenv("DVC_TEST")
-    monkeypatch.delenv("DVC_ANALYTICS", raising=False)
+    monkeypatch.delenv("DVC_NO_ANALYTICS", raising=False)
 
     assert result == analytics.is_enabled()
 
@@ -96,20 +96,23 @@ def test_is_enabled(dvc, config, result, monkeypatch, tmp_global_dir):
     "config, env, result",
     [
         (None, None, True),
-        (None, "true", True),
-        (None, "false", False),
+        (None, "true", False),
+        (None, "false", False),  # only checking if env is set
         ("false", None, False),
-        ("false", "true", True),
+        ("false", "true", False),
         ("false", "false", False),
         ("true", None, True),
-        ("true", "true", True),
-        ("true", "false", False),
+        ("true", "true", False),
+        ("true", "false", False),  # we checking if env is set
     ],
 )
-def test_is_enabled_env(dvc, config, env, result, monkeypatch, tmp_global_dir):
+def test_is_enabled_env_neg(
+    dvc, config, env, result, monkeypatch, tmp_global_dir
+):
     # reset DVC_TEST env var, which affects `is_enabled()`
     monkeypatch.delenv("DVC_TEST")
-    monkeypatch.delenv("DVC_ANALYTICS", raising=False)
+    monkeypatch.delenv("DVC_NO_ANALYTICS", raising=False)
+
     with dvc.config.edit() as conf:
         conf["core"] = {}
 
@@ -120,7 +123,7 @@ def test_is_enabled_env(dvc, config, env, result, monkeypatch, tmp_global_dir):
             conf["core"] = {"analytics": config}
 
     if env is not None:
-        monkeypatch.setenv("DVC_ANALYTICS", env)
+        monkeypatch.setenv("DVC_NO_ANALYTICS", env)
 
     assert result == analytics.is_enabled()
 

tests/unit/utils/serialize/test_yaml.py

@@ -1,6 +1,11 @@
 import pytest
 
-from dvc.utils.serialize import EncodingError, YAMLError, load_yaml, parse_yaml
+from dvc.utils.serialize import (
+    EncodingError,
+    YAMLFileCorruptedError,
+    load_yaml,
+    parse_yaml,
+)
 
 
 def test_parse_yaml_duplicate_key_error():
@@ -10,7 +15,7 @@ def test_parse_yaml_duplicate_key_error():
     mykey:
     - bar
     """
-    with pytest.raises(YAMLError, match='found duplicate key "mykey"'):
+    with pytest.raises(YAMLFileCorruptedError):
         parse_yaml(text, "mypath")