test on 3.11 (#8196)

We don't have aiohttp support yet, due to which we have to install
without cythonized extension.

Similarly, there has been some changes in how dataclasses disallows some
mutable arguments to be set as a class property, due to which
`hydra.conf` fails to import in 3.11.

See https://github.com/python/cpython/pull/29867. For now, I have
patched the module and avoid importing as a pytest-plugin.

dvc-hdfs/pyarrow is not there yet, we'll usually see a release 2-3
months after official reelase.

dvc-hdfs is skipped in 3.11 in extras_requires. `--set-params` will
return a nice error message if hydra cannot be imported in 3.11. I
decided to only patch that in tests.

.github/workflows/tests.yaml

@@ -43,6 +43,9 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-latest, macos-latest]
         pyv: ["3.8", "3.9", "3.10"]
+        include:
+        - {os: ubuntu-latest, pyv: "3.11-dev"}
+
     steps:
     - uses: actions/checkout@v3
       with:
@@ -68,6 +71,8 @@ jobs:
       run: |
         pip install --upgrade pip setuptools wheel
         pip install -e ".[dev]"
+      env:
+        AIOHTTP_NO_EXTENSIONS: ${{ matrix.pyv == '3.11-dev' && '1' }}
     - name: run tests
       timeout-minutes: 40
       run: >-

dvc/repo/experiments/queue/base.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import sys
 from abc import ABC, abstractmethod
 from dataclasses import asdict, dataclass
 from typing import (
@@ -524,7 +525,14 @@ class BaseStashQueue(ABC):
         """
         logger.debug("Using experiment params '%s'", params)
 
-        from dvc.utils.hydra import apply_overrides
+        try:
+            from dvc.utils.hydra import apply_overrides
+        except ValueError:
+            if sys.version_info >= (3, 11):
+                raise DvcException(
+                    "--set-param is not supported in Python >= 3.11"
+                )
+            raise
 
         for path, overrides in params.items():
             apply_overrides(path, overrides)

pyproject.toml

@@ -30,11 +30,11 @@ line_length = 79
 
 [tool.pytest.ini_options]
 log_level = "debug"
-addopts = "-ra"
+addopts = "-ra -p no:hydra_pytest"
 markers = [
     "needs_internet: Might need network access for the tests",
-"vscode: Tests verifying contract between DVC and VSCode plugin",
+    "vscode: Tests verifying contract between DVC and VSCode plugin",
-"studio: Tests verifying contract between DVC and Studio"
+    "studio: Tests verifying contract between DVC and Studio",
 ]
 xfail_strict = true
 

setup.cfg

@@ -62,8 +62,6 @@ install_requires =
     rich>=10.13.0
     pyparsing>=2.4.7
     typing-extensions>=3.7.4
-    fsspec[http]>=2021.10.1
-    aiohttp-retry>=2.4.5
     scmrepo==0.0.25
     dvc-render==0.0.9
     dvc-task==0.1.2
@@ -91,7 +89,7 @@ dev =
 azure = dvc-azure==2.19.0
 gdrive = dvc-gdrive==2.19.0
 gs = dvc-gs==2.19.1
-hdfs = dvc-hdfs==2.19.0
+hdfs = dvc-hdfs==2.19.0; python_version < '3.11'
 oss = dvc-oss==2.19.0
 s3 = dvc-s3==2.19.0
 ssh = dvc-ssh==2.19.0

tests/conftest.py

@@ -250,3 +250,15 @@ def run_copy_metrics(tmp_dir, run_copy):
         return stage
 
     return run
+
+
+@pytest.fixture(autouse=True)
+def mock_hydra_conf(mocker):
+    if sys.version_info < (3, 11):
+        return
+
+    # `hydra.conf` fails to import in 3.11, it raises ValueError due to changes
+    # in dataclasses. See https://github.com/python/cpython/pull/29867.
+    # NOTE: using sentinel here so that any imports from `hydra.conf`
+    # return a mock.
+    sys.modules["hydra.conf"] = mocker.sentinel

tests/func/utils/test_hydra.py

@@ -1,7 +1,6 @@
 import pytest
 
 from dvc.exceptions import InvalidArgumentError
-from dvc.utils.hydra import apply_overrides
 
 
 @pytest.mark.parametrize("suffix", ["yaml", "toml", "json"])
@@ -90,6 +89,8 @@ from dvc.utils.hydra import apply_overrides
     ],
 )
 def test_apply_overrides(tmp_dir, suffix, overrides, expected):
+    from dvc.utils.hydra import apply_overrides
+
     if suffix == "toml":
         if overrides in [
             ["foo=baz"],
@@ -114,6 +115,8 @@ def test_apply_overrides(tmp_dir, suffix, overrides, expected):
     [["foobar=2"], ["lorem=3,2"], ["+lorem=3"], ["foo[0]=bar"]],
 )
 def test_invalid_overrides(tmp_dir, overrides):
+    from dvc.utils.hydra import apply_overrides
+
     params_file = tmp_dir / "params.yaml"
     params_file.dump(
         {"foo": [{"bar": 1}, {"baz": 2}], "goo": {"bag": 3.0}, "lorem": False}

tests/unit/fs/test_fs.py

@@ -1,5 +1,4 @@
 import pytest
-from dvc_hdfs import HDFSFileSystem
 from dvc_http import HTTPFileSystem, HTTPSFileSystem
 from dvc_s3 import S3FileSystem
 from dvc_ssh import SSHFileSystem
@@ -7,25 +6,31 @@ from dvc_ssh import SSHFileSystem
 from dvc.config import RemoteNotFoundError
 from dvc.fs import LocalFileSystem, get_fs_cls, get_fs_config
 
+url_cls_pairs = [
+    ("s3://bucket/path", S3FileSystem),
+    ("ssh://example.com:/dir/path", SSHFileSystem),
+    ("http://example.com/path/to/file", HTTPFileSystem),
+    ("https://example.com/path/to/file", HTTPSFileSystem),
+    ("path/to/file", LocalFileSystem),
+    ("path\\to\\file", LocalFileSystem),
+    ("file", LocalFileSystem),
+    ("./file", LocalFileSystem),
+    (".\\file", LocalFileSystem),
+    ("../file", LocalFileSystem),
+    ("..\\file", LocalFileSystem),
+    ("unknown://path", LocalFileSystem),
+]
 
-@pytest.mark.parametrize(
+
-    "url, cls",
+try:
-    [
+    from dvc_hdfs import HDFSFileSystem
-        ("s3://bucket/path", S3FileSystem),
+
-        ("ssh://example.com:/dir/path", SSHFileSystem),
+    url_cls_pairs += [("hdfs://example.com/dir/path", HDFSFileSystem)]
-        ("hdfs://example.com/dir/path", HDFSFileSystem),
+except ImportError:
-        ("http://example.com/path/to/file", HTTPFileSystem),
+    pass
-        ("https://example.com/path/to/file", HTTPSFileSystem),
+
-        ("path/to/file", LocalFileSystem),
+
-        ("path\\to\\file", LocalFileSystem),
+@pytest.mark.parametrize("url, cls", url_cls_pairs)
-        ("file", LocalFileSystem),
-        ("./file", LocalFileSystem),
-        (".\\file", LocalFileSystem),
-        ("../file", LocalFileSystem),
-        ("..\\file", LocalFileSystem),
-        ("unknown://path", LocalFileSystem),
-    ],
-)
 def test_get_fs_cls(url, cls):
     assert get_fs_cls({"url": url}) == cls