analytics: fix mac not sending reports (#10026)

Bumps [pylint](https://github.com/pylint-dev/pylint) from 2.17.6 to 3.0.1.
- [Release notes](https://github.com/pylint-dev/pylint/releases)
- [Commits](https://github.com/pylint-dev/pylint/compare/v2.17.6...v3.0.1)

---
updated-dependencies:
- dependency-name: pylint
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>

dvc/commands/cache.py

@@ -40,8 +40,11 @@ class CmdCacheDir(CmdConfig):
 class CmdCacheMigrate(CmdBase):
     def run(self):
         from dvc.cachemgr import migrate_2_to_3
+        from dvc.repo.commit import commit_2_to_3
 
         migrate_2_to_3(self.repo, dry=self.args.dry)
+        if self.args.dvc_files:
+            commit_2_to_3(self.repo, dry=self.args.dry)
         return 0
 
 
@@ -102,6 +105,14 @@ def add_parser(subparsers, parent_parser):
         help=CACHE_MIGRATE_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    cache_migrate_parser.add_argument(
+        "--dvc-files",
+        help=(
+            "Migrate entries in all existing DVC files in the repository "
+            "to the DVC 3.0 format."
+        ),
+        action="store_true",
+    )
     cache_migrate_parser.add_argument(
         "--dry",
         help=(

dvc/commands/data_sync.py

@@ -14,6 +14,10 @@ class CmdDataBase(CmdBase):
         from dvc.utils.humanize import get_summary
 
         default_msg = "Everything is up to date."
+
+        if not self.args.remote and not self.repo.config["core"].get("remote"):
+            ui.warn("No remote provided and no default remote set.")
+
         ui.write(get_summary(stats.items()) or default_msg)
 
 

dvc/commands/experiments/apply.py

@@ -4,21 +4,13 @@ import logging
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdExperimentsApply(CmdBase):
     def run(self):
-        if not self.args.force:
-            ui.write(
-                "The --no-force option is deprecated and will be removed in a future"
-                " DVC release. To revert the result of 'exp apply', run:\n"
-                "\n\tgit reset --hard\n"
-                "\tgit stash apply refs/exps/apply/stash\n"
-            )
-        self.repo.experiments.apply(self.args.experiment)
+        self.repo.experiments.apply(self.args.experiment, force=self.args.force)
 
         return 0
 
@@ -33,10 +25,11 @@ def add_parser(experiments_subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     experiments_apply_parser.add_argument(
-        "--no-force",
-        action="store_false",
-        dest="force",
-        help="Fail if this command would overwrite conflicting changes.",
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Do not prompt when removing working directory files.",
     )
     experiments_apply_parser.add_argument(
         "experiment", help="Experiment to be applied."

dvc/commands/experiments/exec_run.py

@@ -49,7 +49,7 @@ def add_parser(experiments_subparsers, parent_parser):
         ),
     )
     exec_run_parser.add_argument(
-        "-M",
+        "-m",
         "--message",
         type=str,
         default=None,

dvc/commands/experiments/rename.py

@@ -55,7 +55,7 @@ def add_parser(experiments_subparsers, parent_parser):
     )
     experiments_rename_parser.add_argument(
         "name",
-        help="Name of new experiment.",
+        help="New name for the experiment.",
         metavar="<name>",
     )
     experiments_rename_parser.add_argument(

dvc/commands/experiments/run.py

@@ -106,3 +106,8 @@ def _add_run_common(parser):
         default=None,
         help="Custom commit message to use when committing the experiment.",
     )
+    parser.add_argument(
+        "-M",  # obsolete
+        dest="message",
+        help=argparse.SUPPRESS,
+    )

dvc/commands/experiments/save.py

@@ -72,10 +72,15 @@ def add_parser(experiments_subparsers, parent_parser):
         metavar="<path>",
     )
     save_parser.add_argument(
-        "-M",
+        "-m",
         "--message",
         type=str,
         default=None,
         help="Custom commit message to use when committing the experiment.",
     )
+    save_parser.add_argument(
+        "-M",  # obsolete
+        dest="message",
+        help=argparse.SUPPRESS,
+    )
     save_parser.set_defaults(func=CmdExperimentsSave)

dvc/commands/gc.py

@@ -81,6 +81,7 @@ class CmdGC(CmdBase):
             rev=self.args.rev,
             num=self.args.num,
             not_in_remote=self.args.not_in_remote,
+            dry=self.args.dry,
         )
         return 0
 
@@ -213,4 +214,10 @@ def add_parser(subparsers, parent_parser):
         ),
         metavar="<paths>",
     )
+    gc_parser.add_argument(
+        "--dry",
+        action="store_true",
+        default=False,
+        help=("Only print what would get removed without actually removing."),
+    )
     gc_parser.set_defaults(func=CmdGC)

dvc/commands/get_url.py

@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
 
 class CmdGetUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.config import Config
         from dvc.repo import Repo
 
         try:
@@ -20,6 +21,7 @@ class CmdGetUrl(CmdBaseNoRepo):
                 jobs=self.args.jobs,
                 force=self.args.force,
                 fs_config=self.args.fs_config,
+                config=Config.from_cwd(),
             )
             return 0
         except DvcException:

dvc/commands/ls_url.py

@@ -11,12 +11,14 @@ logger = logging.getLogger(__name__)
 
 class CmdListUrl(CmdBaseNoRepo):
     def run(self):
+        from dvc.config import Config
         from dvc.repo import Repo
 
         entries = Repo.ls_url(
             self.args.url,
             recursive=self.args.recursive,
             fs_config=self.args.fs_config,
+            config=Config.from_cwd(),
         )
         if entries:
             show_entries(entries, with_color=True, with_size=self.args.size)

dvc/daemon.py

@@ -3,7 +3,6 @@
 import inspect
 import logging
 import os
-import platform
 import sys
 from subprocess import Popen  # nosec B404
 from typing import List
@@ -93,13 +92,8 @@ def _spawn_posix(cmd, env):
     sys.stderr.close()
     os.closerange(0, 3)
 
-    if platform.system() == "Darwin":
-        # workaround for MacOS bug
-        # https://github.com/iterative/dvc/issues/4294
-        _popen(cmd, env=env).communicate()
-    else:
-        os.environ.update(env)
-        main(cmd)
+    os.environ.update(env)
+    main(cmd)
 
     os._exit(0)  # pylint: disable=protected-access
 

dvc/data_cloud.py

@@ -135,7 +135,7 @@ class DataCloud:
 
         remote = self.get_remote(name=name, command=command)
         if remote.fs.version_aware or remote.worktree:
-            raise NoRemoteError(
+            raise RemoteConfigError(
                 f"'{command}' is unsupported for cloud versioned remotes"
             )
         if hash_name in LEGACY_HASH_NAMES:

dvc/fs/__init__.py

@@ -88,11 +88,11 @@ def download(
         return len(to_infos)
 
 
-def parse_external_url(url, config=None):
-    remote_config = dict(config) if config else {}
+def parse_external_url(url, fs_config=None, config=None):
+    remote_config = dict(fs_config) if fs_config else {}
     remote_config["url"] = url
-    fs_cls, fs_config, fs_path = get_cloud_fs({}, **remote_config)
-    fs = fs_cls(**fs_config)
+    fs_cls, resolved_fs_config, fs_path = get_cloud_fs(config, **remote_config)
+    fs = fs_cls(**resolved_fs_config)
     return fs, fs_path
 
 

dvc/output.py

@@ -721,7 +721,7 @@ class Output:
         if self.metric:
             self.verify_metric()
 
-        self._update_legacy_hash_name()
+        self.update_legacy_hash_name()
         if self.use_cache:
             _, self.meta, self.obj = self._build(
                 self.cache,
@@ -745,8 +745,8 @@ class Output:
         self.hash_info = self.obj.hash_info
         self.files = None
 
-    def _update_legacy_hash_name(self):
-        if self.hash_name == "md5-dos2unix" and self.changed_checksum():
+    def update_legacy_hash_name(self, force: bool = False):
+        if self.hash_name == "md5-dos2unix" and (force or self.changed_checksum()):
             self.hash_name = "md5"
 
     def set_exec(self) -> None:
@@ -1391,7 +1391,7 @@ class Output:
             )
 
         assert self.repo
-        self._update_legacy_hash_name()
+        self.update_legacy_hash_name()
         cache = self.cache if self.use_cache else self.local_cache
         assert isinstance(cache, HashFileDB)
 

dvc/repo/__init__.py

@@ -577,6 +577,26 @@ class Repo:
 
         return DVCFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf)
 
+    @cached_property
+    def _btime(self):
+        if not self.tmp_dir:
+            return None
+
+        # Not all python versions/filesystems/platforms provide creation
+        # time (st_birthtime, stx_btime, etc), so we use our own dummy
+        # file and its mtime instead.
+        path = os.path.join(self.tmp_dir, "btime")
+
+        try:
+            with open(path, "x"):  # pylint: disable=unspecified-encoding
+                pass
+        except FileNotFoundError:
+            return None
+        except FileExistsError:
+            pass
+
+        return os.path.getmtime(path)
+
     @cached_property
     def site_cache_dir(self) -> str:
         import getpass
@@ -609,12 +629,17 @@ class Repo:
         # components were changed (useful to prevent newer dvc versions from
         # using older broken cache). Please reset this back to 0 if other parts
         # of the token components are changed.
-        salt = 0
+        salt = 1
+
+        # NOTE: This helps us avoid accidentally reusing cache for repositories
+        # that just happened to be at the same path as old deleted ones.
+        btime = self._btime or getattr(os.stat(root_dir), "st_birthtime", None)
 
         md5 = hashlib.md5(  # noqa: S324  # nosec B324, B303
             str(
                 (
                     root_dir,
+                    btime,
                     getpass.getuser(),
                     version_tuple[0],
                     salt,

dvc/repo/commit.py

@@ -1,6 +1,12 @@
+from typing import TYPE_CHECKING
+
 from dvc import prompt
 
 from . import locked
+from .scm_context import scm_context
+
+if TYPE_CHECKING:
+    from . import Repo
 
 
 def _prepare_message(stage, changes):
@@ -68,3 +74,48 @@ def commit(
         )
         stage.dump(update_pipeline=False)
     return [s.stage for s in stages_info]
+
+
+@locked
+@scm_context
+def commit_2_to_3(repo: "Repo", dry: bool = False):
+    """Force-commit all legacy outputs to use DVC 3.0 hashes."""
+    from dvc.dvcfile import ProjectFile
+    from dvc.ui import ui
+
+    view = repo.index.targets_view(
+        targets=None,
+        outs_filter=lambda o: o.hash_name == "md5-dos2unix",
+        recursive=True,
+    )
+    migrated = set()
+    for out in view.outs:
+        dvcfile = out.stage.dvcfile.relpath
+        if isinstance(out.stage.dvcfile, ProjectFile):
+            # pylint: disable-next=protected-access
+            lockfile = out.stage.dvcfile._lockfile.relpath
+            migrated.add(f"{dvcfile} ({lockfile})")
+        else:
+            migrated.add(dvcfile)
+    if not migrated:
+        ui.write("No DVC files in the repo to migrate to the 3.0 format.")
+        return
+    if dry:
+        ui.write("Entries in following DVC files will be migrated to the 3.0 format:")
+        ui.write("\n".join(sorted(f"\t{file}" for file in migrated)))
+        return
+    for stage, filter_info in view._stage_infos:  # pylint: disable=protected-access
+        outs_filter = view._outs_filter  # pylint: disable=protected-access
+        outs = {
+            out
+            for out in stage.filter_outs(filter_info)
+            if outs_filter is not None and outs_filter(out)
+        }
+        if outs:
+            for out in outs:
+                out.update_legacy_hash_name(force=True)
+            stage.save(allow_missing=True)
+            stage.commit(allow_missing=True, relink=True)
+            if not isinstance(stage.dvcfile, ProjectFile):
+                ui.write(f"Updating DVC file '{stage.dvcfile.relpath}'")
+            stage.dump(update_pipeline=False)

dvc/repo/diff.py

@@ -22,7 +22,7 @@ def _hash(entry):
     return None
 
 
-def _diff(old, new, with_missing=False):
+def _diff(old, new, data_keys, with_missing=False):
     from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME
     from dvc_data.index.diff import diff as idiff
 
@@ -34,12 +34,23 @@ def _diff(old, new, with_missing=False):
         "not in cache": [],
     }
 
+    def meta_cmp_key(meta):
+        if not meta:
+            return meta
+        return meta.isdir
+
     for change in idiff(
         old,
         new,
         with_renames=True,
-        hash_only=True,
+        meta_cmp_key=meta_cmp_key,
+        roots=data_keys,
     ):
+        if (change.old and change.old.isdir and not change.old.hash_info) or (
+            change.new and change.new.isdir and not change.new.hash_info
+        ):
+            continue
+
         if change.typ == ADD:
             ret["added"].append(
                 {
@@ -116,6 +127,7 @@ def diff(
         b_rev = "workspace"
         with_missing = True
 
+    data_keys = set()
     for rev in self.brancher(revs=[a_rev, b_rev]):
         if rev == "workspace" and b_rev != "workspace":
             # brancher always returns workspace, but we only need to compute
@@ -132,6 +144,8 @@ def diff(
             recursive=recursive,
         )
 
+        data_keys.update(view.data_keys.get("repo", set()))
+
         if rev == "workspace":
             from .index import build_data_index
 
@@ -165,4 +179,4 @@ def diff(
         new = indexes[b_rev]
 
     with ui.status("Calculating diff"):
-        return _diff(old, new, with_missing=with_missing)
+        return _diff(old, new, data_keys, with_missing=with_missing)

dvc/repo/experiments/__init__.py

@@ -241,7 +241,7 @@ class Experiments:
                 self.check_baseline(last_applied)
             return last_applied
         except BaselineMismatchError:
-            # If HEAD has moved since the the last applied experiment,
+            # If HEAD has moved since the last applied experiment,
             # the applied experiment is no longer relevant
             self.scm.remove_ref(EXEC_APPLY)
         return None

dvc/repo/experiments/rename.py

@@ -5,7 +5,7 @@ from dvc.repo.experiments.exceptions import (
     ExperimentExistsError,
     UnresolvedExpNamesError,
 )
-from dvc.repo.experiments.utils import resolve_name
+from dvc.repo.experiments.utils import check_ref_format, resolve_name
 from dvc.scm import Git
 
 from .refs import ExpRefInfo
@@ -43,6 +43,7 @@ def rename(
             if repo.scm.get_ref(str(new_ref)) and not force:
                 raise ExperimentExistsError(new_name)
 
+            check_ref_format(repo.scm, new_ref)
             _rename_exp(scm=repo.scm, ref_info=result, new_name=new_name)
             renamed.append(name)
 

dvc/repo/gc.py

@@ -67,6 +67,7 @@ def gc(  # noqa: PLR0913, C901
     rev: Optional[str] = None,
     num: Optional[int] = None,
     not_in_remote: bool = False,
+    dry: bool = False,
 ):
     # require `workspace` to be true to come into effect.
     # assume `workspace` to be enabled if any of `all_tags`, `all_commits`,
@@ -128,8 +129,10 @@ def gc(  # noqa: PLR0913, C901
     for scheme, odb in self.cache.by_scheme():
         if not odb:
             continue
-        removed = ogc(odb, used_obj_ids, jobs=jobs)
-        if not removed:
+        num_removed = ogc(odb, used_obj_ids, jobs=jobs, dry=dry)
+        if num_removed:
+            logger.info("Removed %d objects from %s cache.", num_removed, scheme)
+        else:
             logger.info("No unused '%s' cache to remove.", scheme)
 
     if not cloud:
@@ -137,9 +140,10 @@ def gc(  # noqa: PLR0913, C901
 
     for remote_odb, obj_ids in odb_to_obj_ids.items():
         assert remote_odb is not None
-        removed = ogc(remote_odb, obj_ids, jobs=jobs)
-        if removed:
+        num_removed = ogc(remote_odb, obj_ids, jobs=jobs, dry=dry)
+        if num_removed:
             get_index(remote_odb).clear()
+            logger.info("Removed %d objects from remote.", num_removed)
         else:
             logger.info("No unused cache to remove from remote.")
 

dvc/repo/get_url.py

@@ -6,12 +6,12 @@ from dvc.fs import download, parse_external_url
 from dvc.utils import resolve_output
 
 
-def get_url(url, out=None, *, fs_config=None, jobs=None, force=False):
+def get_url(url, out=None, *, fs_config=None, jobs=None, force=False, config=None):
     out = resolve_output(url, out, force=force)
     out = os.path.abspath(out)
     (out,) = output.loads_from(None, [out], use_cache=False)
 
-    src_fs, src_path = parse_external_url(url, fs_config)
+    src_fs, src_path = parse_external_url(url, fs_config, config=config)
     if not src_fs.exists(src_path):
         raise URLMissingError(url)
     download(src_fs, src_path, out.fs_path, jobs=jobs)

dvc/repo/index.py

@@ -1,5 +1,6 @@
 import logging
 import time
+from collections import defaultdict
 from functools import partial
 from typing import (
     TYPE_CHECKING,
@@ -319,6 +320,22 @@ class Index:
         for stage in self.stages:
             yield from stage.outs
 
+    @cached_property
+    def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
+        by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
+
+        by_workspace["repo"] = set()
+        by_workspace["local"] = set()
+
+        for out in self.outs:
+            if not out.use_cache:
+                continue
+
+            ws, key = out.index_key
+            by_workspace[ws].add(key)
+
+        return dict(by_workspace)
+
     @property
     def decorated_outs(self) -> Iterator["Output"]:
         for output in self.outs:
@@ -359,8 +376,6 @@ class Index:
 
     @cached_property
     def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
-        from collections import defaultdict
-
         by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
 
         by_workspace["repo"] = set()
@@ -377,8 +392,6 @@ class Index:
 
     @cached_property
     def metric_keys(self) -> Dict[str, Set["DataIndexKey"]]:
-        from collections import defaultdict
-
         from .metrics.show import _collect_top_level_metrics
 
         by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
@@ -400,8 +413,6 @@ class Index:
 
     @cached_property
     def plot_keys(self) -> Dict[str, Set["DataIndexKey"]]:
-        from collections import defaultdict
-
         by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
 
         by_workspace["repo"] = set()
@@ -520,8 +531,6 @@ class Index:
         jobs: Optional[int] = None,
         push: bool = False,
     ) -> "ObjectContainer":
-        from collections import defaultdict
-
         used: "ObjectContainer" = defaultdict(set)
         pairs = self.collect_targets(targets, recursive=recursive, with_deps=with_deps)
         for stage, filter_info in pairs:
@@ -640,9 +649,23 @@ class IndexView:
         yield from {out for (out, _) in self._filtered_outs}
 
     @cached_property
-    def _data_prefixes(self) -> Dict[str, "_DataPrefixes"]:
-        from collections import defaultdict
+    def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
+        by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
 
+        by_workspace["repo"] = set()
+        by_workspace["local"] = set()
+
+        for out in self.outs:
+            if not out.use_cache:
+                continue
+
+            ws, key = out.index_key
+            by_workspace[ws].add(key)
+
+        return dict(by_workspace)
+
+    @cached_property
+    def _data_prefixes(self) -> Dict[str, "_DataPrefixes"]:
         prefixes: Dict[str, "_DataPrefixes"] = defaultdict(
             lambda: _DataPrefixes(set(), set())
         )
@@ -660,8 +683,6 @@ class IndexView:
 
     @cached_property
     def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
-        from collections import defaultdict
-
         ret: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
 
         for out, filter_info in self._filtered_outs:
@@ -714,7 +735,7 @@ class IndexView:
         return data
 
 
-def build_data_index(  # noqa: C901
+def build_data_index(  # noqa: C901, PLR0912
     index: Union["Index", "IndexView"],
     path: str,
     fs: "FileSystem",
@@ -777,14 +798,6 @@ def build_data_index(  # noqa: C901
             data.add(entry)
             callback.relative_update(1)
 
-        if compute_hash:
-            tree_meta, tree = build_tree(data, key, name=hash_name)
-            out_entry.meta = tree_meta
-            out_entry.hash_info = tree.hash_info
-            out_entry.loaded = True
-            data.add(out_entry)
-            callback.relative_update(1)
-
     for key in parents:
         parent_path = fs.path.join(path, *key)
         if not fs.exists(parent_path):
@@ -793,6 +806,23 @@ def build_data_index(  # noqa: C901
         data.add(direntry)
         callback.relative_update(1)
 
+    if compute_hash:
+        out_keys = index.out_data_keys.get(workspace, set())
+        data_keys = index.data_keys.get(workspace, set())
+        for key in data_keys.intersection(out_keys):
+            hash_name = _get_entry_hash_name(index, workspace, key)
+
+            out_entry = data.get(key)
+            if not out_entry or not out_entry.isdir:
+                continue
+
+            tree_meta, tree = build_tree(data, key, name=hash_name)
+            out_entry.meta = tree_meta
+            out_entry.hash_info = tree.hash_info
+            out_entry.loaded = True
+            data.add(out_entry)
+            callback.relative_update(1)
+
     return data
 
 
@@ -801,10 +831,14 @@ def _get_entry_hash_name(
 ) -> str:
     from dvc_data.hashfile.hash import DEFAULT_ALGORITHM
 
-    try:
-        src_entry = index.data[workspace][key]
+    for idx in reversed(range(len(key) + 1)):
+        prefix = key[:idx]
+        try:
+            src_entry = index.data[workspace][prefix]
+        except KeyError:
+            continue
+
         if src_entry.hash_info and src_entry.hash_info.name:
             return src_entry.hash_info.name
-    except KeyError:
-        pass
+
     return DEFAULT_ALGORITHM

dvc/repo/ls_url.py

@@ -2,8 +2,8 @@ from dvc.exceptions import URLMissingError
 from dvc.fs import parse_external_url
 
 
-def ls_url(url, *, fs_config=None, recursive=False):
-    fs, fs_path = parse_external_url(url, config=fs_config)
+def ls_url(url, *, fs_config=None, recursive=False, config=None):
+    fs, fs_path = parse_external_url(url, fs_config=fs_config, config=config)
     try:
         info = fs.info(fs_path)
     except FileNotFoundError as exc:

dvc/repo/stage.py

@@ -261,7 +261,7 @@ class StageLoad:
         Args:
             path: if not provided, default `dvc.yaml` is assumed.
             name: required for `dvc.yaml` files, ignored for `.dvc` files.
-            accept_group: if true, all of the the stages generated from `name`
+            accept_group: if true, all of the stages generated from `name`
                 foreach are returned.
             glob: if true, `name` is considered as a glob, which is
                 used to filter list of stages from the given `path`.

dvc/stage/cache.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple
 from funcy import first
 
 from dvc import fs
+from dvc.config import RemoteConfigError
 from dvc.exceptions import CollectCacheError, DvcException
 from dvc.utils import dict_sha256, relpath
 
@@ -274,15 +275,21 @@ class StageCache:
         return ret
 
     def push(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
-        dest_odb = odb or self.repo.cloud.get_remote_odb(
-            remote, "push --run-cache", hash_name="md5-dos2unix"
-        )
+        try:
+            dest_odb = odb or self.repo.cloud.get_remote_odb(
+                remote, "push --run-cache", hash_name="md5-dos2unix"
+            )
+        except RemoteConfigError as e:
+            raise RunCacheNotSupported(e) from e
         return self.transfer(self.repo.cache.legacy, dest_odb)
 
     def pull(self, remote: Optional[str], odb: Optional["ObjectDB"] = None):
-        odb = odb or self.repo.cloud.get_remote_odb(
-            remote, "fetch --run-cache", hash_name="md5-dos2unix"
-        )
+        try:
+            odb = odb or self.repo.cloud.get_remote_odb(
+                remote, "fetch --run-cache", hash_name="md5-dos2unix"
+            )
+        except RemoteConfigError as e:
+            raise RunCacheNotSupported(e) from e
         return self.transfer(odb, self.repo.cache.legacy)
 
     def get_used_objs(self, used_run_cache, *args, **kwargs):

dvc/utils/__init__.py

@@ -205,7 +205,7 @@ def boxify(message, border_color=None):
 
 
 def _visual_width(line):
-    """Get the the number of columns required to display a string"""
+    """Get the number of columns required to display a string"""
 
     return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line))
 

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=2.16.3,<2.17.0",
+    "dvc-data>=2.18.1,<2.19.0",
     "dvc-http>=2.29.0",
     "dvc-render>=0.3.1,<1",
     "dvc-studio-client>=0.13.0,<1",
@@ -44,7 +44,7 @@ dependencies = [
     "flufl.lock>=5,<8",
     "funcy>=1.14",
     "grandalf<1,>=0.7",
-    "gto>=1.3.0,<2",
+    "gto>=1.4.0,<2",
     "hydra-core>=1.1",
     "iterative-telemetry>=0.0.7",
     "networkx>=2.5",
@@ -77,7 +77,7 @@ gs = ["dvc-gs==2.22.1"]
 hdfs = ["dvc-hdfs==2.19"]
 lint = [
     "mypy==1.5.1",
-    "pylint==2.17.6",
+    "pylint==3.0.1",
     "types-colorama",
     "types-psutil",
     "types-pyinstaller",

tests/func/artifacts/test_artifacts.py

@@ -43,7 +43,7 @@ def test_artifacts_read_subdir(tmp_dir, dvc):
 
 def test_artifacts_read_bad_name(tmp_dir, dvc, caplog):
     bad_name_dvcyaml = deepcopy(dvcyaml)
-    bad_name_dvcyaml["artifacts"]["Bad_name"] = {"type": "model", "path": "bad.pkl"}
+    bad_name_dvcyaml["artifacts"]["_bad_name_"] = {"type": "model", "path": "bad.pkl"}
 
     (tmp_dir / "dvc.yaml").dump(bad_name_dvcyaml)
 
@@ -54,7 +54,7 @@ def test_artifacts_read_bad_name(tmp_dir, dvc, caplog):
 
     with caplog.at_level(logging.WARNING):
         assert tmp_dir.dvc.artifacts.read() == {"dvc.yaml": artifacts}
-        assert "Can't use 'Bad_name' as artifact name (ID)" in caplog.text
+        assert "Can't use '_bad_name_' as artifact name (ID)" in caplog.text
 
 
 def test_artifacts_add_subdir(tmp_dir, dvc):

tests/func/experiments/test_rename.py

@@ -1,5 +1,6 @@
 import pytest
 
+from dvc.exceptions import InvalidArgumentError
 from dvc.repo.experiments.exceptions import (
     ExperimentExistsError,
     UnresolvedExpNamesError,
@@ -31,3 +32,13 @@ def test_existing_name(dvc, exp_stage):
         dvc.experiments.rename("second-name", "first-name")
 
     dvc.experiments.rename("second-name", "first-name", force=True)
+
+
+def test_invalid_name(dvc, exp_stage):
+    dvc.experiments.run(exp_stage.addressing, name="test-name", params=["foo=1"])
+
+    with pytest.raises(InvalidArgumentError):
+        dvc.experiments.rename("invalid*name", "test-name")
+
+    with pytest.raises(InvalidArgumentError):
+        dvc.experiments.rename("invalid/name", "test-name")

tests/func/repro/test_repro.py

@@ -442,7 +442,7 @@ def test_repro_list_of_commands_raise_and_stops_after_failure(tmp_dir, dvc, mult
     assert not (tmp_dir / "bar").exists()
 
 
-def test_repro_pulls_mising_data_source(tmp_dir, dvc, mocker, local_remote):
+def test_repro_pulls_missing_data_source(tmp_dir, dvc, mocker, local_remote):
     (foo,) = tmp_dir.dvc_gen("foo", "foo")
 
     dvc.push()
@@ -454,7 +454,7 @@ def test_repro_pulls_mising_data_source(tmp_dir, dvc, mocker, local_remote):
     assert dvc.reproduce(pull=True)
 
 
-def test_repro_pulls_mising_import(tmp_dir, dvc, mocker, erepo_dir, local_remote):
+def test_repro_pulls_missing_import(tmp_dir, dvc, mocker, erepo_dir, local_remote):
     with erepo_dir.chdir():
         erepo_dir.dvc_gen("foo", "foo", commit="first")
 

tests/func/repro/test_repro_pull.py

@@ -4,7 +4,7 @@ from dvc.stage.cache import RunCacheNotSupported
 from dvc.utils.fs import remove
 
 
-def test_repro_pulls_mising_data_source(tmp_dir, dvc, mocker, local_remote):
+def test_repro_pulls_missing_data_source(tmp_dir, dvc, mocker, local_remote):
     (foo,) = tmp_dir.dvc_gen("foo", "foo")
 
     dvc.push()
@@ -16,7 +16,7 @@ def test_repro_pulls_mising_data_source(tmp_dir, dvc, mocker, local_remote):
     assert dvc.reproduce(pull=True)
 
 
-def test_repro_pulls_mising_import(tmp_dir, dvc, mocker, erepo_dir, local_remote):
+def test_repro_pulls_missing_import(tmp_dir, dvc, mocker, erepo_dir, local_remote):
     with erepo_dir.chdir():
         erepo_dir.dvc_gen("foo", "foo", commit="first")
 

tests/func/test_diff.py

@@ -668,3 +668,59 @@ def test_rename_multiple_files_same_hashes(tmp_dir, scm, dvc):
             },
         ],
     }
+
+
+def test_diff_granular(tmp_dir, dvc, scm):
+    tmp_dir.gen(
+        {
+            "dir": {
+                "data": {
+                    "subdir": {"subfoo": "subfoo", "subbar": "subbar"},
+                    "foo": "foo",
+                    "bar": "bar",
+                },
+            },
+        }
+    )
+
+    dvc.add(os.path.join("dir", "data"))
+    scm.add(os.path.join("dir", "data.dvc"))
+    scm.add(os.path.join("dir", ".gitignore"))
+    scm.commit("data")
+
+    assert dvc.diff() == {}
+
+    (tmp_dir / "dir" / "data" / "subdir" / "new").write_text("new")
+
+    assert dvc.diff() == {
+        "added": [
+            {
+                "hash": "22af645d1859cb5ca6da0c484f1f37ea",
+                "path": os.path.join("dir", "data", "subdir", "new"),
+            }
+        ],
+        "deleted": [],
+        "modified": [
+            {
+                "hash": {
+                    "new": "efa5b20d5f935dcc5555b26db6e19b76.dir",
+                    "old": "1aca2c799df82929bbdd976557975546.dir",
+                },
+                "path": os.path.join("dir", "data", ""),
+            }
+        ],
+        "not in cache": [],
+        "renamed": [],
+    }
+    assert dvc.diff(targets=[os.path.join("dir", "data", "subdir")]) == {
+        "added": [
+            {
+                "hash": "22af645d1859cb5ca6da0c484f1f37ea",
+                "path": os.path.join("dir", "data", "subdir", "new"),
+            }
+        ],
+        "deleted": [],
+        "modified": [],
+        "not in cache": [],
+        "renamed": [],
+    }

tests/func/test_gc.py

@@ -36,14 +36,14 @@ def test_gc_api(dvc, good_and_bad_cache):
     dvc.gc(workspace=True)
     odb = dvc.cache.local
     good_cache, bad_cache = good_and_bad_cache
-    assert set(odb.oids_exist([*good_cache, *bad_cache])) == good_cache
+    assert set(odb.all()) == good_cache
 
 
 def test_gc_cli(dvc, good_and_bad_cache):
     assert main(["gc", "-wf"]) == 0
     odb = dvc.cache.local
     good_cache, bad_cache = good_and_bad_cache
-    assert set(odb.oids_exist([*good_cache, *bad_cache])) == good_cache
+    assert set(odb.all()) == good_cache
 
 
 def test_gc_branches_tags(tmp_dir, dvc, scm):
@@ -423,3 +423,19 @@ def test_gc_cloud_remote_field(tmp_dir, scm, dvc, mocker, make_remote):
     mocked_remove = mocker.spy(LocalFileSystem, "remove")
     dvc.gc(workspace=True, cloud=True)
     assert len(mocked_remove.mock_calls) == 2  # local and other_remote
+
+
+def test_gc_dry(dvc, good_and_bad_cache):
+    dvc.gc(workspace=True, dry=True)
+    odb = dvc.cache.local
+    good_cache, bad_cache = good_and_bad_cache
+    assert set(odb.all()) != good_cache
+
+
+def test_gc_logging(caplog, dvc, good_and_bad_cache):
+    with caplog.at_level(logging.INFO, logger="dvc"):
+        dvc.gc(workspace=True)
+
+    assert "Removed 3 objects from repo cache." in caplog.text
+    assert "No unused 'local' cache to remove." in caplog.text
+    assert "No unused 'legacy' cache to remove." in caplog.text

tests/func/test_get_url.py

@@ -3,6 +3,7 @@ import os
 
 import pytest
 
+from dvc.config import Config
 from dvc.exceptions import FileExistsLocallyError, URLMissingError
 from dvc.repo import Repo
 from dvc.testing.workspace_tests import TestGetUrl as _TestGetUrl
@@ -66,3 +67,12 @@ def test_get_url_nonexistent(tmp_dir):
 
 class TestGetUrl(_TestGetUrl):
     pass
+
+
+def test_get_url_config(tmp_dir, dvc, make_remote):
+    remote_path = make_remote("myremote", default=False, typ="local")
+    (remote_path / "foo").write_text("foo")
+    (remote_path / "bar").write_text("bar")
+
+    Repo.get_url("remote://myremote/foo", config=Config.from_cwd())
+    assert (tmp_dir / "foo").read_text() == "foo"

tests/func/test_ls_url.py

@@ -1,5 +1,23 @@
+from dvc.config import Config
+from dvc.repo import Repo
 from dvc.testing.workspace_tests import TestLsUrl as _TestLsUrl
 
 
 class TestLsUrl(_TestLsUrl):
     pass
+
+
+def test_ls_url_config(dvc, make_remote):
+    remote_path = make_remote("myremote", default=False, typ="local")
+    (remote_path / "foo").write_text("foo")
+    (remote_path / "bar").write_text("bar")
+
+    actual = sorted(
+        Repo.ls_url("remote://myremote", config=Config.from_cwd()),
+        key=lambda entry: entry["path"],
+    )
+    expected = [
+        {"isdir": False, "path": "bar", "size": 3},
+        {"isdir": False, "path": "foo", "size": 3},
+    ]
+    assert actual == expected

tests/func/test_run_cache.py

@@ -4,7 +4,7 @@ import pytest
 from funcy import first
 
 from dvc.dvcfile import LOCK_FILE
-from dvc.stage.cache import _get_stage_hash
+from dvc.stage.cache import RunCacheNotSupported, _get_stage_hash
 from dvc.utils.fs import remove
 
 
@@ -229,3 +229,15 @@ def test_restore_pull(tmp_dir, dvc, run_copy, mocker, local_remote):
     assert (tmp_dir / "bar").exists()
     assert not (tmp_dir / "foo").unlink()
     assert (tmp_dir / LOCK_FILE).exists()
+
+
+def test_push_pull_unsupported(tmp_dir, dvc, mocker, run_copy):
+    tmp_dir.gen("foo", "foo")
+    run_copy("foo", "bar", name="copy-foo-bar")
+    mocker.patch.object(
+        dvc.cloud, "get_remote_odb", side_effect=RunCacheNotSupported("foo")
+    )
+    with pytest.raises(RunCacheNotSupported):
+        dvc.push(run_cache=True)
+    with pytest.raises(RunCacheNotSupported):
+        dvc.pull(run_cache=True)

tests/unit/command/test_experiments.py

@@ -27,7 +27,19 @@ def test_experiments_apply(dvc, scm, mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with(cmd.repo, "exp_rev")
+    m.assert_called_once_with(cmd.repo, "exp_rev", force=False)
+
+
+def test_experiments_apply_force(dvc, scm, mocker):
+    cli_args = parse_args(["experiments", "apply", "exp_rev", "--force"])
+    assert cli_args.func == CmdExperimentsApply
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.experiments.apply.apply", return_value={})
+
+    assert cmd.run() == 0
+
+    m.assert_called_once_with(cmd.repo, "exp_rev", force=True)
 
 
 def test_experiments_diff(dvc, scm, mocker):
@@ -138,6 +150,27 @@ def test_experiments_run(dvc, scm, mocker):
     cmd.repo.experiments.run.assert_called_with(**default_arguments)
 
 
+@pytest.mark.parametrize("flag", ["-m", "-M", "--message"])
+def test_experiments_run_message(dvc, scm, mocker, flag):
+    default_arguments = {
+        "params": [],
+        "name": None,
+        "queue": False,
+        "run_all": False,
+        "jobs": 1,
+        "tmp_dir": False,
+        "copy_paths": [],
+        "message": "mymessage",
+    }
+    default_arguments.update(repro_arguments)
+
+    cmd = CmdExperimentsRun(parse_args(["exp", "run", flag, "mymessage"]))
+    mocker.patch.object(cmd.repo, "reproduce")
+    mocker.patch.object(cmd.repo.experiments, "run")
+    cmd.run()
+    cmd.repo.experiments.run.assert_called_with(**default_arguments)
+
+
 def test_experiments_branch(dvc, scm, mocker):
     m = mocker.patch("dvc.repo.experiments.branch.branch", return_value={})
 
@@ -467,6 +500,25 @@ def test_experiments_save(dvc, scm, mocker):
     )
 
 
+@pytest.mark.parametrize("flag", ["-m", "-M", "--message"])
+def test_experiments_save_message(dvc, scm, mocker, flag):
+    cli_args = parse_args(["exp", "save", flag, "custom commit message"])
+    assert cli_args.func == CmdExperimentsSave
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.experiments.save.save", return_value="acabb")
+
+    assert cmd.run() == 0
+
+    m.assert_called_once_with(
+        cmd.repo,
+        name=None,
+        force=False,
+        include_untracked=[],
+        message="custom commit message",
+    )
+
+
 def test_experiments_clean(dvc, scm, mocker):
     cli_args = parse_args(["experiments", "clean"])
     assert cli_args.func == CmdExperimentsClean

tests/unit/command/test_gc.py

@@ -22,6 +22,7 @@ def test_(dvc, scm, mocker):
             "--force",
             "--jobs",
             "3",
+            "--dry",
             "--projects",
             "project1",
             "project2",
@@ -49,6 +50,7 @@ def test_(dvc, scm, mocker):
         rev=None,
         num=None,
         not_in_remote=False,
+        dry=True,
     )
 
     cli_args = parse_args(["gc"])

tests/unit/command/test_get_url.py

@@ -1,8 +1,9 @@
 from dvc.cli import parse_args
 from dvc.commands.get_url import CmdGetUrl
+from dvc.config import Config
 
 
-def test_get_url(mocker):
+def test_get_url(mocker, M):
     cli_args = parse_args(["get-url", "src", "out", "-j", "5"])
     assert cli_args.func == CmdGetUrl
 
@@ -11,4 +12,11 @@ def test_get_url(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", out="out", jobs=5, force=False, fs_config=None)
+    m.assert_called_once_with(
+        "src",
+        out="out",
+        jobs=5,
+        force=False,
+        fs_config=None,
+        config=M.instance_of(Config),
+    )

tests/unit/command/test_ls_url.py

@@ -1,8 +1,9 @@
 from dvc.cli import parse_args
 from dvc.commands.ls_url import CmdListUrl
+from dvc.config import Config
 
 
-def test_ls_url(mocker):
+def test_ls_url(mocker, M):
     cli_args = parse_args(["ls-url", "src"])
     assert cli_args.func == CmdListUrl
     cmd = cli_args.func(cli_args)
@@ -10,10 +11,12 @@ def test_ls_url(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", recursive=False, fs_config=None)
+    m.assert_called_once_with(
+        "src", recursive=False, fs_config=None, config=M.instance_of(Config)
+    )
 
 
-def test_recursive(mocker):
+def test_recursive(mocker, M):
     cli_args = parse_args(["ls-url", "-R", "src"])
     assert cli_args.func == CmdListUrl
     cmd = cli_args.func(cli_args)
@@ -21,4 +24,6 @@ def test_recursive(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", recursive=True, fs_config=None)
+    m.assert_called_once_with(
+        "src", recursive=True, fs_config=None, config=M.instance_of(Config)
+    )

tests/unit/repo/experiments/test_executor_status.py

@@ -107,7 +107,7 @@ def test_workspace_executor_failed_status(dvc, scm, failed_exp_stage, queue_type
         assert executor_info.status == TaskStatus.FAILED
 
 
-def test_executor_status_compatiblity():
+def test_executor_status_compatibility():
     data = {
         "git_url": "file:///Users/home",
         "baseline_rev": "123",

tests/unit/repo/test_repo.py

@@ -127,7 +127,7 @@ def test_branch_config(tmp_dir, scm):
         dvc.close()
 
 
-def test_dynamic_cache_initalization(tmp_dir, scm):
+def test_dynamic_cache_initialization(tmp_dir, scm):
     dvc = Repo.init()
     with dvc.config.edit() as conf:
         conf["cache"]["ssh"] = "foo"