dvc: bump to 2.6.2

We've used updater for years now to notify our users ASAP about new
releases. These days dvc is more stable and we've actually been
receiving numerous complains about updater being too annoying and
even introduced a config flag to turn it off, which a lot of people do
and which defeats its purpose. Instead of introducing some complex logic
that would notify more rarely and once for the user instead of once for
every dvc project they have - this PR just turns it off for everything,
except `dvc doctor/version`, which is the command that people use when
they are running into issues and that will give them a hint to just try
out the newest version first, as it might be already fixed.

This PR is a more of a compromise, to get the most out of updater while
removing a pain point of it being too annoying, without investing much
time into it and not having to maintain complex unnecessary logic. We
might get back to introducing something more intricate in the future,
if the need arises.

Fixes #5912

.github/workflows/tests.yaml

@@ -42,13 +42,14 @@ jobs:
       matrix:
         os: [ubuntu-18.04, windows-2019, macos-10.15]
         pyv: ["3.6", "3.7", "3.8", "3.9"]
-# macos 11.0 workers take too long to start up
-#        include:
-#          - os: "macos-11.0"
-#            pyv: "3.9"
+        include:
+          - os: "macos-10.15"
+            pyv: "3.10.0-rc.1"
         exclude:
           - os: macos-10.15
             pyv: "3.9"
+          - os: macos-10.15
+            pyv: "3.7"
 
     steps:
     - uses: actions/[email protected]
@@ -68,10 +69,14 @@ jobs:
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
         restore-keys: |
           ${{ runner.os }}-pip-
+    - name: install libgit2 on Python3.10 in macOS
+      if: matrix.pyv == '3.10.0-rc.1' && matrix.os == 'macos-10.15'
+      run: |
+        brew install libgit2
     - name: install
       if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
-        pip install wheel
+        pip install --upgrade pip setuptools wheel
         pip install -e ".[all,tests]"
     - name: install temporary dependencies
       run: |

dvc/analytics.py

@@ -2,6 +2,8 @@ import json
 import logging
 import os
 
+from .env import DVC_ANALYTICS
+
 logger = logging.getLogger(__name__)
 
 
@@ -42,9 +44,12 @@ def is_enabled():
     if env2bool("DVC_TEST"):
         return False
 
-    enabled = to_bool(
-        Config(validate=False).get("core", {}).get("analytics", "true")
-    )
+    enabled = env2bool(DVC_ANALYTICS, None)
+    if enabled is None:
+        enabled = to_bool(
+            Config(validate=False).get("core", {}).get("analytics", "true")
+        )
+
     logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
     return enabled

dvc/command/base.py

@@ -33,17 +33,12 @@ class CmdBase(ABC):
 
     def __init__(self, args):
         from dvc.repo import Repo
-        from dvc.updater import Updater
 
         os.chdir(args.cd)
 
         self.repo = Repo(uninitialized=self.UNINITIALIZED)
         self.config = self.repo.config
         self.args = args
-        if self.repo.tmp_dir:
-            hardlink_lock = self.config["core"].get("hardlink_lock", False)
-            updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
-            updater.check()
 
     def do_run(self):
         with self.repo:

dvc/command/version.py

@@ -10,9 +10,12 @@ logger = logging.getLogger(__name__)
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
         from dvc.info import get_dvc_info
+        from dvc.updater import notify_updates
 
         dvc_info = get_dvc_info()
         ui.write(dvc_info, force=True)
+
+        notify_updates()
         return 0
 
 

dvc/env.py

@@ -9,3 +9,4 @@ DVCLIVE_RESUME = "DVCLIVE_RESUME"
 DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY"
 DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE"
 DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH"
+DVC_ANALYTICS = "DVC_ANALYTICS"

dvc/fs/fsspec_wrapper.py

@@ -217,7 +217,7 @@ class ObjectFSWrapper(FSSpecWrapper):
             files = self.fs.find(path, detail=detail)
 
         if detail:
-            files = files.values()
+            files = list(files.values())
 
         # When calling find() on a file, it returns the same file in a list.
         # For object-based storages, the same behavior applies to empty

dvc/repo/fetch.py

@@ -61,7 +61,8 @@ def fetch(
 
     for odb, obj_ids in sorted(
         used.items(),
-        key=lambda item: item[0] and item[0].fs.scheme == Schemes.MEMORY,
+        key=lambda item: item[0] is not None
+        and item[0].fs.scheme == Schemes.MEMORY,
     ):
         d, f = _fetch(
             self,

dvc/updater.py

@@ -2,17 +2,20 @@ import logging
 import os
 import sys
 import time
+from typing import TYPE_CHECKING, Optional
 
-import colorama
 from packaging import version
 
 from dvc import __version__
 from dvc.utils.pkg import PKG
 
+if TYPE_CHECKING:
+    from rich.text import Text
+
 logger = logging.getLogger(__name__)
 
 
-class Updater:  # pragma: no cover
+class Updater:
     URL = "https://updater.dvc.org"
     UPDATER_FILE = "updater"
     TIMEOUT = 24 * 60 * 60  # every day
@@ -105,57 +108,58 @@ class Updater:  # pragma: no cover
         with open(self.updater_file, "w+") as fobj:
             json.dump(info, fobj)
 
-    def _notify(self, latest):
+    def _notify(self, latest: str, pkg: Optional[str] = PKG) -> None:
+        from dvc.ui import ui
+
         if not sys.stdout.isatty():
             return
 
-        message = (
-            "Update available {red}{current}{reset} -> {green}{latest}{reset}"
-            + "\n"
-            + self._get_update_instructions()
-        ).format(
-            red=colorama.Fore.RED,
-            reset=colorama.Fore.RESET,
-            green=colorama.Fore.GREEN,
-            yellow=colorama.Fore.YELLOW,
-            blue=colorama.Fore.BLUE,
-            current=self.current,
-            latest=latest,
+        message = self._get_message(latest, pkg=pkg)
+        return ui.error_write(message, styled=True)
+
+    def _get_message(
+        self,
+        latest: str,
+        current: str = None,
+        color: str = "yellow",
+        pkg: Optional[str] = None,
+    ) -> "Text":
+        from rich.text import Text
+
+        current = current or self.current
+        update_message = Text.from_markup(
+            f"You are using dvc version [bold]{current}[/]; "
+            f"however, version [bold]{latest}[/] is available."
+        )
+        instruction = Text.from_markup(self._get_update_instructions(pkg=pkg))
+        return Text.assemble(
+            "\n", update_message, "\n", instruction, style=color
         )
 
-        from dvc.utils import boxify
+    def _get_update_instructions(self, pkg: Optional[str] = None) -> str:
+        if pkg in ("osxpkg", "exe", "binary"):
+            return (
+                "To upgrade, uninstall dvc and reinstall from "
+                "[blue]https://dvc.org[/]."
+            )
 
-        logger.info(boxify(message, border_color="yellow"))
-
-    def _get_update_instructions(self):
         instructions = {
-            "pip": "Run `{yellow}pip{reset} install dvc "
-            "{blue}--upgrade{reset}`",
-            "rpm": "Run `{yellow}yum{reset} update dvc`",
-            "brew": "Run `{yellow}brew{reset} upgrade dvc`",
-            "deb": (
-                "Run `{yellow}apt-get{reset} install"
-                " {blue}--only-upgrade{reset} dvc`"
-            ),
-            "binary": (
-                "To upgrade follow these steps:\n"
-                "1. Uninstall dvc binary\n"
-                "2. Go to {blue}https://dvc.org{reset}\n"
-                "3. Download and install new binary"
-            ),
-            "conda": "Run `{yellow}conda{reset} update dvc`",
-            "choco": "Run `{yellow}choco{reset} upgrade dvc`",
-            None: (
-                "Find the latest release at\n"
-                "{blue}https://github.com/iterative/dvc/releases/latest{reset}"
-            ),
+            "pip": "pip install --upgrade dvc",
+            "rpm": "yum update dvc",
+            "brew": "brew upgrade dvc",
+            "deb": "apt-get install --only-upgrade dvc",
+            "conda": "conda update dvc",
+            "choco": "choco upgrade dvc",
         }
 
-        package_manager = PKG
-        if package_manager in ("osxpkg", "exe"):
-            package_manager = "binary"
+        if pkg not in instructions:
+            return (
+                "Find the latest release at "
+                "[blue]https://github.com/iterative/dvc/releases/latest[/]."
+            )
 
-        return instructions[package_manager]
+        instruction = instructions[pkg]
+        return f"To upgrade, run '{instruction}'."
 
     def is_enabled(self):
         from dvc.config import Config, to_bool
@@ -167,3 +171,14 @@ class Updater:  # pragma: no cover
             "Check for update is {}abled.".format("en" if enabled else "dis")
         )
         return enabled
+
+
+def notify_updates():
+    from contextlib import suppress
+
+    from dvc.repo import NotDvcRepoError, Repo
+
+    with suppress(NotDvcRepoError), Repo() as repo:
+        hardlink_lock = repo.config["core"].get("hardlink_lock", False)
+        updater = Updater(repo.tmp_dir, hardlink_lock=hardlink_lock)
+        updater.check()

dvc/utils/html.py

@@ -7,9 +7,9 @@ PAGE_HTML = """<!DOCTYPE html>
 <html>
 <head>
     <title>DVC Plot</title>
-    <script src="https://cdn.jsdelivr.net/npm/vega@5.10.0"></script>
-    <script src="https://cdn.jsdelivr.net/npm/vega-lite@4.8.1"></script>
-    <script src="https://cdn.jsdelivr.net/npm/vega-embed@6.5.1"></script>
+    <script src="https://cdn.jsdelivr.net/npm/vega@5.20.2"></script>
+    <script src="https://cdn.jsdelivr.net/npm/vega-lite@5.1.0"></script>
+    <script src="https://cdn.jsdelivr.net/npm/vega-embed@6.18.2"></script>
 </head>
 <body>
     {plot_divs}

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.6.0"
+_BASE_VERSION = "2.6.2"
 
 
 def _generate_version(base_version):

setup.py

@@ -71,7 +71,7 @@ install_requires = [
     "zc.lockfile>=1.2.1",
     "flufl.lock>=3.2,<4",
     "win-unicode-console>=0.5; sys_platform == 'win32'",
-    "pywin32>=225; sys_platform == 'win32'",
+    "pywin32>=225; sys_platform == 'win32' and python_version < '3.10'",
     "networkx>=2.5",
     "psutil>=5.8.0",
     "pydot>=1.2.4",
@@ -96,15 +96,13 @@ install_requires = [
 # Extra dependencies for remote integrations
 
 gs = ["gcsfs==2021.7.0"]
-gdrive = ["pydrive2[fsspec]>=1.9.0"]
+gdrive = ["pydrive2[fsspec]>=1.9.1"]
 s3 = ["s3fs==2021.7.0", "aiobotocore[boto3]>1.0.1"]
 azure = ["adlfs==2021.7.1", "azure-identity>=1.4.0", "knack"]
-# https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["ossfs==2021.7.5"]
 ssh = ["sshfs>=2021.7.1"]
 
-# Remove the env marker if/when pyarrow is available for Python3.9
-hdfs = ["pyarrow>=2.0.0"]
+hdfs = ["pyarrow>=2.0.0; python_version < '3.10'"]
 webhdfs = ["hdfs==2.5.8"]
 webdav = ["webdav4>=0.9.0"]
 # gssapi should not be included in all_remotes, because it doesn't have wheels

test_requirements.txt

@@ -25,7 +25,7 @@ google-cloud-storage==1.42.0
 urllib3==1.26.6
 
 # required by collective.checkdocs
-Pygments==2.9.0
+Pygments==2.10.0
 collective.checkdocs==0.2
 pydocstyle==6.1.1
 jaraco.windows==5.6.0

tests/dir_helpers.py

@@ -47,6 +47,7 @@ from global repo template to creating everything inplace, which:
 
 import os
 import pathlib
+import sys
 from contextlib import contextmanager
 from textwrap import dedent
 
@@ -81,12 +82,17 @@ class TmpDir(pathlib.Path):
             cls = (  # pylint: disable=self-cls-assignment
                 WindowsTmpDir if os.name == "nt" else PosixTmpDir
             )
-        self = cls._from_parts(args, init=False)
+        # init parameter and `_init` method has been removed in Python 3.10.
+        kw = {"init": False} if sys.version_info < (3, 10) else {}
+        self = cls._from_parts(  # pylint: disable=unexpected-keyword-arg
+            args, **kw
+        )
         if not self._flavour.is_supported:
             raise NotImplementedError(
                 f"cannot instantiate {cls.__name__!r} on your system"
             )
-        self._init()
+        if sys.version_info < (3, 10):
+            self._init()  # pylint: disable=no-member
         return self
 
     def init(self, *, scm=False, dvc=False, subdir=False):

tests/func/test_data_cloud.py

@@ -350,6 +350,27 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir):
     assert (tmp_dir / "new_dir" / "bar").read_text() == "bar"
 
 
+def test_pull_external_dvc_imports_mixed(
+    tmp_dir, dvc, scm, erepo_dir, local_remote
+):
+    with erepo_dir.chdir():
+        erepo_dir.dvc_gen("foo", "foo", commit="first")
+        os.remove("foo")
+
+    # imported: foo
+    dvc.imp(os.fspath(erepo_dir), "foo")
+
+    # local-object: bar
+    tmp_dir.dvc_gen("bar", "bar")
+    dvc.push("bar")
+
+    clean(["foo", "bar"], dvc)
+
+    assert dvc.pull()["fetched"] == 2
+    assert (tmp_dir / "foo").read_text() == "foo"
+    assert (tmp_dir / "bar").read_text() == "bar"
+
+
 def clean(outs, dvc=None):
     from tests.utils import clean_staging
 

tests/func/test_fs.py

@@ -335,6 +335,10 @@ def test_fs_find(dvc, cloud):
     assert {
         os.path.basename(file_key) for file_key in fs.find(path_info / "data")
     } == {"foo", "baz", "quux"}
+    assert {
+        os.path.basename(file_info["name"])
+        for file_info in fs.find(path_info / "data", detail=True)
+    } == {"foo", "baz", "quux"}
 
 
 @pytest.mark.parametrize(

tests/remotes/hdfs.py

@@ -2,6 +2,7 @@ import locale
 import os
 import platform
 import subprocess
+import sys
 import uuid
 from contextlib import contextmanager
 from pathlib import Path
@@ -74,6 +75,9 @@ class HDFS(Base, URLInfo):  # pylint: disable=abstract-method
 def hadoop(test_config):
     test_config.requires("hdfs")
 
+    if sys.version_info >= (3, 10):
+        pytest.skip("pyarrow is not available for 3.10 yet.")
+
     if platform.system() != "Linux":
         pytest.skip("only supported on Linux")
 
@@ -240,11 +244,16 @@ class FakeHadoopFileSystem:
 
 
 @pytest.fixture
-def hdfs(mocker):
+def hdfs(test_config, mocker):
     # Windows might not have Visual C++ Redistributable for Visual Studio
     # 2015 installed, which will result in the following error:
     # "The pyarrow installation is not built with support for
     # 'HadoopFileSystem'"
+    test_config.requires("hdfs")
+
+    if sys.version_info >= (3, 10):
+        pytest.skip("pyarrow is not available for 3.10 yet.")
+
     mocker.patch("pyarrow.fs._not_imported", [])
     mocker.patch(
         "pyarrow.fs.HadoopFileSystem", FakeHadoopFileSystem, create=True

tests/unit/test_analytics.py

@@ -87,6 +87,40 @@ def test_is_enabled(dvc, config, result, monkeypatch, tmp_global_dir):
 
     # reset DVC_TEST env var, which affects `is_enabled()`
     monkeypatch.delenv("DVC_TEST")
+    monkeypatch.delenv("DVC_ANALYTICS", raising=False)
+
+    assert result == analytics.is_enabled()
+
+
+@pytest.mark.parametrize(
+    "config, env, result",
+    [
+        (None, None, True),
+        (None, "true", True),
+        (None, "false", False),
+        ("false", None, False),
+        ("false", "true", True),
+        ("false", "false", False),
+        ("true", None, True),
+        ("true", "true", True),
+        ("true", "false", False),
+    ],
+)
+def test_is_enabled_env(dvc, config, env, result, monkeypatch, tmp_global_dir):
+    # reset DVC_TEST env var, which affects `is_enabled()`
+    monkeypatch.delenv("DVC_TEST")
+    monkeypatch.delenv("DVC_ANALYTICS", raising=False)
+    with dvc.config.edit() as conf:
+        conf["core"] = {}
+
+    assert analytics.is_enabled()
+
+    if config is not None:
+        with dvc.config.edit() as conf:
+            conf["core"] = {"analytics": config}
+
+    if env is not None:
+        monkeypatch.setenv("DVC_ANALYTICS", env)
 
     assert result == analytics.is_enabled()
 

tests/unit/test_updater.py

@@ -8,6 +8,7 @@ import pytest
 
 from dvc import __version__
 from dvc.updater import Updater
+from tests.func.parsing.test_errors import escape_ansi
 
 
 @pytest.fixture
@@ -85,19 +86,26 @@ def test_check_update_respect_config(mock_check, result, updater, mocker):
     ],
     ids=["uptodate", "behind", "ahead"],
 )
-def test_check_updates(mock_tty, updater, caplog, current, latest, notify):
+def test_check_updates(mocker, capsys, updater, current, latest, notify):
+    mocker.patch("sys.stdout.isatty", return_value=True)
+
     updater.current = current
     with open(updater.updater_file, "w+") as f:
         json.dump({"version": latest}, f)
 
-    caplog.clear()
-    with caplog.at_level(logging.INFO, logger="dvc.updater"):
-        updater.check()
+    updater.check()
+    out, err = capsys.readouterr()
+    expected_message = (
+        (
+            f"You are using dvc version {current}; "
+            f"however, version {latest} is available.\n"
+        )
+        if notify
+        else ""
+    )
 
-    if notify:
-        assert f"Update available {current} -> {latest}" in caplog.text
-    else:
-        assert not caplog.text
+    assert expected_message in escape_ansi(err)
+    assert not out
 
 
 def test_check_refetches_each_day(mock_tty, updater, caplog, mocker):
@@ -138,3 +146,45 @@ def test_check(mock_check, updater):
     updater.check()
 
     assert mock_check.call_count == 3
+
+
+@pytest.mark.parametrize(
+    "pkg, instruction",
+    [
+        ("pip", "To upgrade, run 'pip install --upgrade dvc'."),
+        ("rpm", "To upgrade, run 'yum update dvc'."),
+        ("brew", "To upgrade, run 'brew upgrade dvc'."),
+        ("deb", "To upgrade, run 'apt-get install --only-upgrade dvc'."),
+        ("conda", "To upgrade, run 'conda update dvc'."),
+        ("choco", "To upgrade, run 'choco upgrade dvc'."),
+        (
+            "osxpkg",
+            "To upgrade, uninstall dvc and reinstall from https://dvc.org.",
+        ),
+        (
+            "exe",
+            "To upgrade, uninstall dvc and reinstall from https://dvc.org.",
+        ),
+        (
+            "binary",
+            "To upgrade, uninstall dvc and reinstall from https://dvc.org.",
+        ),
+        (
+            None,
+            "Find the latest release at "
+            "https://github.com/iterative/dvc/releases/latest.",
+        ),
+        (
+            "unknown",
+            "Find the latest release at "
+            "https://github.com/iterative/dvc/releases/latest.",
+        ),
+    ],
+)
+def test_notify_message(updater, pkg, instruction):
+    update_message = (
+        "You are using dvc version 0.0.2; however, version 0.0.3 is available."
+    )
+
+    message = updater._get_message("0.0.3", current="0.0.2", pkg=pkg)
+    assert message.plain.splitlines() == ["", update_message, instruction]