deps: bump scmrepo to 0.1.3

bump pyinstaller packages python version to 3.10

As 3.11 is now released, let's bump pyinstaller packages to 3.10 (we always keep them 1 version behind).

.codecov.yml

@@ -1,10 +0,0 @@
-comment: false
-coverage:
-  status:
-    project:
-      default:
-        target: 90
-        threshold: 10%
-    patch:
-      default:
-        threshold: 25%

.coveragerc

@@ -1,16 +0,0 @@
-[run]
-branch = True
-source = dvc
-omit =
-    *__main__.py
-
-[report]
-exclude_lines =
-    if __name__ == .__main__.:
-    if typing.TYPE_CHECKING:
-    if TYPE_CHECKING:
-    pass
-    raise NotImplementedError
-    # pragma: no cover
-    #  pragma: no cover
-    \.\.\.$

.dvcignore

@@ -1,4 +1,3 @@
 /tests
 /dvc
 /.github
-/bin

.flake8

@@ -0,0 +1,12 @@
+[flake8]
+ignore=
+    E203, # Whitespace before ':'
+    E266, # Too many leading '#' for block comment
+    W503, # Line break occurred before a binary operator
+    P1,  # unindexed parameters in the str.format, see:
+    # https://pypi.org/project/flake8-string-format/
+max_line_length = 79
+max-complexity = 15
+select = B,C,E,F,W,T4,B902,T,P
+show_source = true
+count = true

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,7 +8,7 @@ about: Create a bug report to help us improve DVC
 <!--
 ## Issue name
 
-Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug. 
+Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug.
 
 Example: `repro: doesn't detect input changes`
 -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,4 +2,3 @@
 name: "\U0001F680 Feature Request"
 about: Suggest an idea for this project
 ---
-

.github/codecov.yml

@@ -0,0 +1,12 @@
+codecov:
+  notify:
+    wait_for_ci: true
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 2%
+    patch: off
+github_checks:
+  annotations: false

.github/release.yml

@@ -0,0 +1,16 @@
+changelog:
+  exclude:
+    labels: ["skip-changelog"]
+  categories:
+    - title: 🛠 Breaking Changes
+      labels: ["breaking-change"]
+    - title: 🚀 New Features and Enhancements
+      labels: ["feature", "enhancement", "ui"]
+    - title: ⚡ Optimizations
+      labels: ["optimize", "performance"]
+    - title: 🐛 Bug Fixes
+      labels: ["bugfix", "bug"]
+    - title: 🔨 Maintenance
+      labels: ["maintenance", "refactoring", "chore", "build", "ci"]
+    - title: Other Changes
+      labels: ["*"]

.github/workflows/packages.yaml

@@ -7,8 +7,13 @@ on:
     types: [released, prereleased]
   workflow_dispatch:
 
+permissions:  # added using https://github.com/step-security/secure-workflows
+  contents: read
+
 jobs:
   binary:
+    permissions:
+      contents: write  # for actions/upload-release-asset to upload release asset
     strategy:
       matrix:
         include:
@@ -24,13 +29,13 @@ jobs:
       with:
           fetch-depth: 0
 
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v4
       with:
-        python-version: 3.9
+        python-version: "3.10"
         cache: pip
         cache-dependency-path: |
-          setup.cfg
+          pyproject.toml
           scripts/build-requirements.txt
 
     - name: Set up Ruby 2.6
@@ -45,7 +50,7 @@ jobs:
 
     - name: Install deps
       run: |
-        pip install -U wheel pip setuptools
+        pip install --upgrade pip wheel setuptools
         pip install .[all]
         pip install -r scripts/build-requirements.txt
 
@@ -77,21 +82,31 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: 3.8
-        cache: pip
-        cache-dependency-path: setup.cfg
 
     - name: Install
+      run: pip install --upgrade pip wheel
+
+    - name: Force version for Test PyPI uploads
+      if: ${{ !startsWith(github.ref, 'refs/tags') }}
       run: |
-        pip install -U wheel pip build setuptools
-        pip install ".[all,tests]"
+        pip install setuptools_scm
+        echo version=$(python -m setuptools_scm | awk -F+ '{print $1}' | tail -1) >> $GITHUB_ENV
 
     - name: Build packages
-      run: |
-        pip install -U pip
-        ./scripts/build_package.sh
+      run: ./scripts/build_package.sh
+      env:
+        SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.version }}
 
     - name: Publish packages to PyPI
       if: github.event_name == 'release'
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.PYPI_TOKEN }}
+
+    - name: Publish to Test PyPI
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.TEST_PYPI_TOKEN }}
+        repository_url: https://test.pypi.org/legacy/
+        skip_existing: true

.github/workflows/tests.yaml

@@ -17,6 +17,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+permissions:  # added using https://github.com/step-security/secure-workflows
+  contents: read
+
 jobs:
   lint:
     timeout-minutes: 10
@@ -29,13 +32,12 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: 3.8
+        cache: pip
+        cache-dependency-path: pyproject.toml
     - name: Install requirements
       run: |
-        pip install --upgrade pip setuptools wheel
-        pip install collective.checkdocs==0.2
+        pip install --upgrade pip wheel
         pip install ".[dev]"
-    - name: Check README
-      run: python setup.py checkdocs
     - uses: pre-commit/[email protected]
   tests:
     timeout-minutes: 45
@@ -43,13 +45,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-latest, macos-latest]
+        os: [ubuntu-20.04, windows-latest]
         pyv: ["3.8", "3.9", "3.10"]
-        pytest-filter: 
+        pytest-filter:
         - "import or plot or live or experiment"
         - "not (import or plot or live or experiment)"
         include:
-        - {os: ubuntu-latest, pyv: "3.11-dev"}
+        - {os: ubuntu-latest, pyv: "3.11"}
+        - {os: macos-latest, pyv: "3.8"}
 
     steps:
     - uses: actions/checkout@v3
@@ -60,25 +63,30 @@ jobs:
       with:
         python-version: ${{ matrix.pyv }}
         cache: pip
-        cache-dependency-path: setup.cfg
+        cache-dependency-path: pyproject.toml
     - name: install
-      if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
-        pip install --upgrade pip setuptools wheel
+        pip install --upgrade pip wheel
         pip install -e ".[dev]"
     - name: run tests
-      timeout-minutes: 40
-      run: pytest -nauto -ra --durations 100 --cov=dvc --cov-report=xml --cov-report=term -k "${{ matrix.pytest-filter }}"
+      timeout-minutes: 20
+      run: pytest -nauto --durations 100 --cov --cov-report=xml --cov-report=term -k "${{ matrix.pytest-filter }}"
     - name: upload coverage report
       uses: codecov/codecov-action@v3
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
+  check:
+    if: always()
+    needs: [lint, tests]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}
   notify:
     if: github.ref == 'refs/heads/main' && failure()
-    needs:
-      - lint
-      - tests
+    needs: [lint, tests]
     runs-on: ubuntu-latest
     steps:
     - name: Slack Notification

.pre-commit-config.yaml

@@ -1,39 +1,62 @@
+default_language_version:
+  python: python3
+
 ci:
   skip: [mypy, pylint, dvc-pre-commit]
 
 repos:
-  - hooks:
-      - id: black
-        language_version: python3
-    repo: https://github.com/ambv/black
-    rev: 22.10.0
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.3.0
     hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-executables-have-shebangs
+      - id: check-json
+      - id: check-merge-conflict
+        exclude: "tests/func/utils/test_strict_yaml.py"
+        args: ['--assume-in-merge']
       - id: check-toml
       - id: check-yaml
-  - hooks:
-      - id: codespell
-        args: 
-          - --ignore-words-list
-          - ba,datas,fo,uptodate,cachable,falsy
-    repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
-  - hooks:
-      - id: isort
-        language_version: python3
-    repo: https://github.com/timothycrosley/isort
+      - id: debug-statements
+        exclude: "dvc/_debug.py"
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: sort-simple-yaml
+      - id: trailing-whitespace
+  - repo: https://github.com/lovesegfault/beautysh
+    rev: v6.2.1
+    hooks:
+      - id: beautysh
+        args: ["-i", "2"]
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.2.0
+    hooks:
+      - id: pyupgrade
+        args:
+          - --py38-plus
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
+  - repo: https://github.com/PyCQA/isort
     rev: 5.10.1
-  - hooks:
+    hooks:
+      - id: isort
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.2
+    hooks:
+      - id: codespell
+        additional_dependencies: ["tomli"]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
+    hooks:
       - id: flake8
-        language_version: python3
         additional_dependencies:
+          - flake8-broken-line
           - flake8-bugbear
           - flake8-comprehensions
           - flake8-debugger
           - flake8-string-format
-    repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
   - repo: local
     hooks:
     - id: mypy
@@ -47,14 +70,6 @@ repos:
       entry: pylint
       language: system
       types: [python]
-  - hooks:
-      - args:
-          - -i
-          - "2"
-        id: beautysh
-        language_version: python3
-    repo: https://github.com/lovesegfault/beautysh
-    rev: v6.2.1
   - hooks:
       - args:
           - git-hook

CODE_OF_CONDUCT.md

@@ -74,4 +74,3 @@ available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.ht
 
 For answers to common questions about this code of conduct, see
 https://www.contributor-covenant.org/faq
-

MANIFEST.in

@@ -1,3 +0,0 @@
-include LICENSE
-recursive-include requirements *.txt
-recursive-include dvc/repo/plots/templates *.json

bin/dvc

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-PYTHONPATH=$DVC_HOME python -m dvc $@

bin/dvc.bat

@@ -1,3 +0,0 @@
-@echo OFF
-set PYTHONPATH=%DVC_HOME%
-python -m dvc %*

dvc/__init__.py

@@ -4,6 +4,6 @@ DVC
 Make your data science projects reproducible and shareable.
 """
 import dvc.logger
-from dvc.version import __version__  # noqa: F401
+from dvc.version import __version__, version_tuple  # noqa: F401
 
 dvc.logger.setup()

dvc/machine/backend/base.py

@@ -26,7 +26,7 @@ class BaseMachineBackend(ABC):
     ) -> Iterator[dict]:
         """Iterate over status of all instances of the specified machine."""
 
-    def close(self):
+    def close(self):  # noqa: B027
         pass
 
     @abstractmethod

dvc/parsing/__init__.py

@@ -36,7 +36,7 @@ from .interpolate import (
 )
 
 if TYPE_CHECKING:
-    from typing_extensions import NoReturn
+    from typing import NoReturn
 
     from dvc.repo import Repo
 

dvc/parsing/interpolate.py

@@ -9,10 +9,9 @@ from dvc.exceptions import DvcException
 from dvc.utils.flatten import flatten
 
 if typing.TYPE_CHECKING:
-    from typing import List, Match
+    from typing import List, Match, NoReturn
 
     from pyparsing import ParseException
-    from typing_extensions import NoReturn
 
     from .context import Context
 

dvc/repo/commit.py

@@ -56,10 +56,13 @@ def commit(
     ]
     for stage_info in stages_info:
         stage = stage_info.stage
-        changes = stage.changed_entries()
-        if any(changes):
-            prompt_to_commit(stage, changes, force=force)
+        if force:
             stage.save(allow_missing=allow_missing)
+        else:
+            changes = stage.changed_entries()
+            if any(changes):
+                prompt_to_commit(stage, changes, force=force)
+                stage.save(allow_missing=allow_missing)
         stage.commit(
             filter_info=stage_info.filter_info, allow_missing=allow_missing
         )

dvc/repo/experiments/queue/base.py

@@ -269,6 +269,7 @@ class BaseStashQueue(ABC):
                 finish any active experiments before shutting down.
         """
 
+    @abstractmethod
     def logs(
         self,
         rev: str,

dvc/repo/experiments/run.py

@@ -1,8 +1,9 @@
 import logging
+import sys
 from typing import Dict, Iterable, Optional
 
 from dvc.dependency.param import ParamsDependency
-from dvc.exceptions import InvalidArgumentError
+from dvc.exceptions import DvcException, InvalidArgumentError
 from dvc.repo import locked
 from dvc.ui import ui
 from dvc.utils.cli_parse import to_path_overrides
@@ -28,7 +29,12 @@ def run(
     Returns a dict mapping new experiment SHAs to the results
     of `repro` for that experiment.
     """
-    from dvc.utils.hydra import to_hydra_overrides
+    try:
+        from dvc.utils.hydra import to_hydra_overrides
+    except ValueError:
+        if sys.version_info >= (3, 11):
+            raise DvcException("exp run is not supported in Python >= 3.11")
+        raise
 
     if run_all:
         entries = list(repo.experiments.celery_queue.iter_queued())

dvc/testing/cloud.py

@@ -65,7 +65,7 @@ class Cloud(ABC):
         self._gen(struct)
         return struct.keys()
 
-    def close(self):
+    def close(self):  # noqa: B027
         pass
 
     @staticmethod

dvc/utils/serialize/_common.py

@@ -1,10 +1,17 @@
 """Common utilities for serialize."""
 import os
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Any, Callable, ContextManager, Dict, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    ContextManager,
+    Dict,
+    Protocol,
+    Union,
+)
 
 from funcy import reraise
-from typing_extensions import Protocol
 
 from dvc.exceptions import DvcException
 

dvc/version.py

@@ -4,3 +4,4 @@ try:
     from ._dvc_version import version_tuple  # noqa: F401
 except ImportError:
     __version__ = "UNKNOWN"
+    version_tuple = (0, 0, __version__)  # type: ignore[assignment]

pyproject.toml

@@ -1,7 +1,140 @@
 [build-system]
-requires = ["setuptools>=48", "setuptools_scm[toml]>=7"]
+requires = ["setuptools>=61", "setuptools_scm[toml]>=7"]
 build-backend = "setuptools.build_meta"
 
+[project]
+name = "dvc"
+description = "Git for data scientists - manage your code and data together"
+readme = "README.rst"
+keywords = [
+    "data-science",
+    "data-version-control",
+    "machine-learning",
+    "git",
+    "developer-tools",
+    "reproducibility",
+    "collaboration",
+    "ai",
+]
+license = { text = "Apache License 2.0" }
+authors = [{ name = "Dmitry Petrov", email = "[email protected]" }]
+maintainers = [{ name = "Iterative", email = "[email protected]" }]
+requires-python = ">=3.8"
+dependencies = [
+    "colorama>=0.3.9",
+    "configobj>=5.0.6",
+    "voluptuous>=0.11.7",
+    "requests>=2.22.0",
+    # See https://github.com/bdcht/grandalf/issues/26
+    "grandalf==0.6",
+    "distro>=1.3.0",
+    "appdirs>=1.4.3",
+    "ruamel.yaml>=0.17.11",
+    "tomlkit>=0.11.1",
+    "funcy>=1.14",
+    "pathspec>=0.9.0,<0.10.0",
+    "shortuuid>=0.5.0",
+    "tqdm>=4.63.1,<5",
+    "packaging>=19.0",
+    "zc.lockfile>=1.2.1",
+    "flufl.lock>=5",
+    "networkx>=2.5",
+    "psutil>=5.8.0",
+    "pydot>=1.2.4",
+    "importlib-resources>=5.2.2; python_version < '3.9'",
+    "flatten_dict>=0.4.1,<1",
+    "tabulate>=0.8.7",
+    "pygtrie>=2.3.2",
+    "dpath>=2.0.2,<3",
+    "shtab>=1.3.4,<2",
+    "rich>=10.13.0",
+    "pyparsing>=2.4.7",
+    "typing-extensions>=3.7.4",
+    "scmrepo==0.1.3",
+    "dvc-render==0.0.12",
+    "dvc-task==0.1.4",
+    "dvclive>=0.10.0",
+    "dvc-data==0.25.2",
+    "dvc-http==2.27.2",
+    "hydra-core>=1.1.0",
+    "iterative-telemetry==0.0.5",
+]
+dynamic = ["version"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+
+[project.optional-dependencies]
+azure = ["dvc-azure==2.20.4"]
+gdrive = ["dvc-gdrive==2.19.0"]
+gs = ["dvc-gs==2.19.1"]
+hdfs = ["dvc-hdfs==2.19.0; python_version < '3.11'"]
+oss = ["dvc-oss==2.19.0"]
+s3 = ["dvc-s3==2.20.1"]
+ssh = ["dvc-ssh==2.20.0"]
+ssh_gssapi = ["dvc-ssh[gssapi]==2.20.0"]
+webdav = ["dvc-webdav==2.19.0"]
+webhdfs = ["dvc-webhdfs==2.19.0"]
+webhdfs_kerberos = ["dvc-webhdfs[kerberos]==2.19.0"]
+terraform = ["tpi[ssh]>=2.1.0"]
+testing = ["pytest-test-utils==0.0.8"]
+tests = [
+    "tpi[ssh]>=2.1.0",
+    "dvc-ssh==2.20.0",
+    "pytest-test-utils==0.0.8",
+    # Test requirements
+    "pytest==7.2.0",
+    "pytest-cov==4.0.0",
+    "pytest-xdist==3.0.2",
+    "pytest-mock==3.10.0",
+    "pytest-lazy-fixture==0.6.3",
+    # https://github.com/docker/docker-py/issues/2902
+    "pytest-docker==0.11.0; python_version < '3.10' or sys_platform != 'win32'",
+    "flaky==3.7.0",
+    "pytest-timeout==2.1.0",
+    "filelock==3.8.0",
+    # optional dependencies
+    "pywin32>=225; sys_platform == 'win32'",
+    "dvclive[image]>=0.7.3",
+    "beautifulsoup4==4.11.1",
+    # pylint requirements
+    "pylint==2.15.5",
+    # we use this to suppress pytest-related false positives in our tests.
+    "pylint-pytest==1.1.2",
+    # we use this to suppress some messages in tests, eg: foo/bar naming,
+    # and, protected method calls in our tests
+    "pylint-plugin-utils==0.7",
+    # type-checking
+    "mypy==0.982",
+    "types-requests>=2.27.15",
+    "types-tabulate>=0.8.6",
+    "types-toml>=0.10.4",
+]
+all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
+dev = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs,tests]"]
+
+[project.urls]
+Documentation = "https://dvc.org/doc"
+"Bug Tracker" = "https://github.com/iterative/dvc/issues"
+Source = "https://github.com/iterative/dvc"
+
+[project.scripts]
+dvc = "dvc.cli:main"
+
+[project.entry-points."fsspec.specs"]
+dvc = "dvc.api:DVCFileSystem"
+
+[tool.setuptools]
+license-files = ["LICENSE"]
+
+[tool.setuptools.packages]
+find = {namespaces = false, exclude=["tests", "tests.*"]}
+
 [tool.setuptools_scm]
 write_to = "dvc/_dvc_version.py"
 
@@ -30,13 +163,33 @@ line_length = 79
 
 [tool.pytest.ini_options]
 log_level = "debug"
-addopts = "-ra -p no:hydra_pytest"
+addopts = "-ra -p no:hydra_pytest --cov-config pyproject.toml"
 markers = [
     "needs_internet: Might need network access for the tests",
     "vscode: Tests verifying contract between DVC and VSCode plugin",
     "studio: Tests verifying contract between DVC and Studio",
 ]
 xfail_strict = true
+testpaths = ["tests"]
+
+[tool.coverage.run]
+branch = true
+source = ["dvc", "tests"]
+
+[tool.coverage.paths]
+source = ["dvc"]
+
+[tool.coverage.report]
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "if __name__ == .__main__.:",
+    "if typing.TYPE_CHECKING:",
+    "if TYPE_CHECKING:",
+    "raise NotImplementedError",
+    "raise AssertionError",
+    "@overload",
+]
 
 [tool.mypy]
 # Error output
@@ -81,3 +234,6 @@ signature-mutators = ["funcy.decorators.decorator"]
 [tool.pylint.variables]
 dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_"
 ignored-argument-names = "_.*|^ignored_|^unused_|args|kwargs"
+
+[tool.codespell]
+ignore-words-list = "ba,datas,fo,uptodate,cachable,falsy"

scripts/build-requirements.txt

@@ -1,3 +1,3 @@
 # NOTE: using strict version due to
 # https://github.com/iterative/dvc/issues/7949
-PyInstaller==5.0
+PyInstaller==5.6.2

scripts/build.py

@@ -1,7 +1,5 @@
 import argparse
-import importlib.util
 import pathlib
-import textwrap
 from subprocess import STDOUT, check_call
 
 path = pathlib.Path(__file__).parent.absolute()
@@ -25,22 +23,8 @@ args = parser.parse_args()
 
 (dvc / "utils" / "build.py").write_text(f'PKG = "{args.pkg}"')
 
-# Autogenerate version, similar to what we do in setup.py
-spec = importlib.util.spec_from_file_location(
-    "dvc.version", dvc / "version.py"
-)
-dvc_version = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(dvc_version)
-version = dvc_version.__version__
-
-(dvc / "version.py").write_text(
-    textwrap.dedent(
-        f"""\
-        # AUTOGENERATED at build time by scripts/build.py
-        __version__ = "{version}"
-    """
-    )
-)
+if not (dvc / "_dvc_version.py").exists():
+    raise Exception("no version info found")
 
 check_call(
     ["python", "build.py"],

scripts/build_package.sh

@@ -10,5 +10,6 @@ fi
 
 echo 'PKG = "pip"' >dvc/utils/build.py
 
-python -m pip install -U build setuptools>=58.2
-python -m build --sdist --wheel --outdir dist/
+python -m pip install build twine
+python -m build
+twine check dist/*

scripts/innosetup/addSymLinkPermissions.ps1

@@ -24,8 +24,8 @@ function addSymLinkPermissions($accountToAdd){
     Write-Host "Account SID: $($sidstr)" -ForegroundColor DarkCyan
     $tmp = [System.IO.Path]::GetTempFileName()
     Write-Host "Export current Local Security Policy" -ForegroundColor DarkCyan
-    secedit.exe /export /cfg "$($tmp)" 
-    $c = Get-Content -Path $tmp 
+    secedit.exe /export /cfg "$($tmp)"
+    $c = Get-Content -Path $tmp
     $currentSetting = ""
     foreach($s in $c) {
         if( $s -like "SECreateSymbolicLinkPrivilege*") {
@@ -35,7 +35,7 @@ function addSymLinkPermissions($accountToAdd){
     }
     if( $currentSetting -notlike "*$($sidstr)*" ) {
         Write-Host "Need to add permissions to SymLink" -ForegroundColor Yellow
-        
+
         Write-Host "Modify Setting ""Create SymLink""" -ForegroundColor DarkCyan
 
         if( [string]::IsNullOrEmpty($currentSetting) ) {
@@ -58,8 +58,8 @@ SECreateSymbolicLinkPrivilege = $($currentSetting)
         $outfile | Set-Content -Path $tmp2 -Encoding Unicode -Force
         Push-Location (Split-Path $tmp2)
         try {
-            secedit.exe /configure /db "secedit.sdb" /cfg "$($tmp2)" /areas USER_RIGHTS 
-        } finally { 
+            secedit.exe /configure /db "secedit.sdb" /cfg "$($tmp2)" /areas USER_RIGHTS
+        } finally {
             Pop-Location
         }
     } else {

scripts/pypi_upload.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -e
-
-if [ ! -d "dvc" ]; then
-  echo "Please run this script from repository root"
-  exit 1
-fi
-
-rm -rf dist/
-pip install twine
-python -m pip install -U build setuptools>=58.2
-python -m build --sdist --wheel --outdir dist/
-twine upload dist/*

setup.cfg

@@ -1,166 +0,0 @@
-[metadata]
-description = Git for data scientists - manage your code and data together
-name = dvc
-long_description = file: README.rst
-long_description_content_type = text/x-rst
-license = Apache License 2.0
-license_file = LICENSE
-url = http://dvc.org
-project_urls =
-    Documentation = https://dvc.org/doc
-    Source = https://github.com/iterative/dvc
-download_url = https://github.com/iterative/dvc
-author = Dmitry Petrov
-author_email = [email protected]
-maintainer = Iterative
-maintainer_email = [email protected]
-keywords = data-science, data-version-control, machine-learning, git
-    developer-tools, reproducibility, collaboration, ai
-classifiers =
-    Development Status :: 4 - Beta
-    Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.9
-    Programming Language :: Python :: 3.10
-    Programming Language :: Python :: 3.11
-
-[options]
-setup_requires =
-    setuptools>=48
-    setuptools_scm[toml]>=7
-
-python_requires = >=3.8
-zip_safe = False
-packages = find:
-include_package_data = True
-install_requires =
-    colorama>=0.3.9
-    configobj>=5.0.6
-    voluptuous>=0.11.7
-    requests>=2.22.0
-    # See https://github.com/bdcht/grandalf/issues/26
-    grandalf==0.6
-    distro>=1.3.0
-    appdirs>=1.4.3
-    ruamel.yaml>=0.17.11
-    tomlkit>=0.11.1
-    funcy>=1.14
-    pathspec>=0.9.0,<0.10.0
-    shortuuid>=0.5.0
-    tqdm>=4.63.1,<5
-    packaging>=19.0
-    zc.lockfile>=1.2.1
-    flufl.lock>=5
-    networkx>=2.5
-    psutil>=5.8.0
-    pydot>=1.2.4
-    importlib-resources>=5.2.2; python_version < '3.9'
-    flatten_dict>=0.4.1,<1
-    tabulate>=0.8.7
-    pygtrie>=2.3.2
-    dpath>=2.0.2,<3
-    shtab>=1.3.4,<2
-    rich>=10.13.0
-    pyparsing>=2.4.7
-    typing-extensions>=3.7.4
-    scmrepo==0.1.2
-    dvc-render==0.0.12
-    dvc-task==0.1.4
-    dvclive>=0.10.0
-    dvc-data==0.22.0
-    dvc-http==2.27.2
-    hydra-core>=1.1.0
-    iterative-telemetry==0.0.5
-
-[options.extras_require]
-all =
-    %(azure)s
-    %(gdrive)s
-    %(gs)s
-    %(hdfs)s
-    %(oss)s
-    %(s3)s
-    %(ssh)s
-    %(webdav)s
-    %(webhdfs)s
-dev =
-    %(all)s
-    %(tests)s
-    dvc-data[cli]
-# Extra dependencies for remote integrations
-azure = dvc-azure==2.20.4
-gdrive = dvc-gdrive==2.19.0
-gs = dvc-gs==2.19.1
-hdfs =
-    dvc-hdfs==2.19.0; python_version < '3.11'
-oss = dvc-oss==2.19.0
-s3 = dvc-s3==2.20.1
-ssh = dvc-ssh==2.20.0
-# gssapi should not be included in all_remotes, because it doesn't have wheels
-# for Linux, so it will fail to compile if user doesn't have all the
-# requirements, including kerberos itself.
-ssh_gssapi = dvc-ssh[gssapi]==2.20.0
-webdav = dvc-webdav==2.19.0
-webhdfs = dvc-webhdfs==2.19.0
-# requests-kerberos requires krb5 & gssapi, which does not provide wheels Linux
-webhdfs_kerberos = dvc-webhdfs[kerberos]==2.19.0
-terraform = tpi[ssh]>=2.1.0
-testing =
-    pytest-test-utils==0.0.8
-tests =
-    %(terraform)s
-    dvc-ssh==2.20.0
-    # Test requirements
-    pytest==7.1.3
-    pytest-cov==4.0.0
-    pytest-xdist==2.5.0
-    pytest-mock==3.10.0
-    pytest-lazy-fixture==0.6.3
-    %(testing)s
-    # https://github.com/docker/docker-py/issues/2902
-    pytest-docker==0.11.0; python_version < '3.10' or sys_platform != 'win32'
-    flaky==3.7.0
-    pytest-timeout==2.1.0
-    filelock==3.8.0
-    # pylint requirements
-    pylint==2.15.4
-    # we use this to suppress pytest-related false positives in our tests.
-    pylint-pytest==1.1.2
-    # we use this to suppress some messages in tests, eg: foo/bar naming,
-    # and, protected method calls in our tests
-    pylint-plugin-utils==0.7
-    # type-checking
-    mypy==0.982
-    types-requests>=2.27.15
-    types-tabulate>=0.8.6
-    types-toml>=0.10.4
-    # optional dependencies
-    pywin32>=225; sys_platform == 'win32'
-    dvclive[image]>=0.7.3
-    beautifulsoup4==4.11.1
-
-[options.packages.find]
-exclude =
-    dvc.testing
-    dvc.testing.*
-    tests
-    tests.*
-
-[options.entry_points]
-console_scripts =
-    dvc = dvc.cli:main
-fsspec.specs =
-    dvc = dvc.api:DVCFileSystem
-
-[flake8]
-ignore=
-    E203, # Whitespace before ':'
-    E266, # Too many leading '#' for block comment
-    W503, # Line break occurred before a binary operator
-    P1,  # unindexed parameters in the str.format, see:
-    # https://pypi.org/project/flake8-string-format/
-max_line_length = 79
-max-complexity = 15
-select = B,C,E,F,W,T4,B902,T,P
-show_source = true
-count = true

setup.py

@@ -1,3 +1,3 @@
 from setuptools import setup
 
-setup()
+setup(name="dvc")

tests/__init__.py

@@ -1,41 +1,13 @@
 import os
-import sys
 
 # Increasing fd ulimit for tests
 if os.name == "nt":
-    import subprocess
-
     try:
         import win32file  # pylint: disable=import-error
     except ImportError:
         pass
     else:
         win32file._setmaxstdio(4096)
-
-    # Workaround for two bugs:
-    #
-    # 1) gitpython-developers/GitPython#546
-    # GitPython leaves git cat-file --batch/--batch-check processes that are
-    # not cleaned up correctly, so Popen._active list has their defunct
-    # process handles, that it is not able to cleanup because of bug 2)
-    #
-    # 2) https://bugs.python.org/issue37380
-    # subprocess.Popen._internal_poll on windows is getting
-    #
-    # 	OSError: [WinError 6] The handle is invalid
-    #
-    # exception, which it doesn't ignore and so Popen is not able to cleanup
-    # old processes and that prevents it from creating any new processes at
-    # all, which results in our tests failing whenever they try to use Popen.
-    # This patch was released in 3.9.0 and backported to some earlier
-    # versions.
-    if sys.version_info < (3, 9, 0):
-
-        def noop():
-            pass
-
-        subprocess._cleanup = noop
-        subprocess._active = None
 else:
     import resource  # pylint: disable=import-error
 

tests/__main__.py

@@ -1,20 +1,6 @@
-# Needs investigation, pylint was running tests
-# Could be `pylint_pytest` plugin, skipping check for this file for now
-# pylint: skip-file
+if __name__ == "__main__":
+    import sys
 
-import os
-import sys
-from subprocess import check_call
+    import pytest
 
-REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-os.chdir(REPO_ROOT)
-
-os.putenv(
-    "PATH", ":".join([os.path.join(REPO_ROOT, "bin"), os.getenv("PATH")])
-)
-os.putenv("DVC_HOME", REPO_ROOT)
-
-opts = "-v -n=auto --dist loadscope --cov=dvc --durations=50"
-params = " ".join(sys.argv[1:])
-cmd = f"{sys.executable} -m pytest {opts} {params}"
-check_call(cmd, shell=True)
+    sys.exit(pytest.main(["-v", "-n=auto", *sys.argv[1:]]))

tests/basic_env.py

@@ -94,7 +94,7 @@ class TestDirFixture:
         self._popd()
         try:
             remove(self._root_dir)
-        except OSError as exc:
+        except OSError as exc:  # pragma: no cover
             # pylint: disable=no-member
             # We ignore this under Windows with a warning because it happened
             # to be really hard to trace all not properly closed files.
@@ -149,12 +149,6 @@ class TestDvcGitFixture(TestGitFixture):
 # NOTE: Inheritance order in the classes below is important.
 
 
-class TestDir(TestDirFixture, TestCase):
-    def __init__(self, methodName):
-        TestDirFixture.__init__(self)
-        TestCase.__init__(self, methodName)
-
-
 class TestDvc(TestDvcFixture, TestCase):
     def __init__(self, methodName):
         TestDvcFixture.__init__(self)

tests/func/experiments/test_set_params.py

@@ -17,7 +17,7 @@ from ..utils.test_hydra import hydra_setup
 def test_modify_params(params_repo, dvc, changes, expected):
     dvc.experiments.run(params=changes)
     # pylint: disable=unspecified-encoding
-    with open("params.yaml", mode="r") as fobj:
+    with open("params.yaml") as fobj:
         assert fobj.read().strip() == expected
 
 

tests/func/test_gc.py

@@ -12,7 +12,7 @@ from dvc.fs import LocalFileSystem
 from dvc.repo import Repo as DvcRepo
 from dvc.utils.fs import remove
 from dvc_data.hashfile.db.local import LocalHashFileDB
-from tests.basic_env import TestDir, TestDvcGit
+from tests.basic_env import TestDvcGit
 
 
 class TestGC(TestDvcGit):
@@ -122,7 +122,7 @@ class TestGCMultipleDvcRepos(TestDvcGit):
 
     def setUp(self):
         super().setUp()
-        self.additional_path = TestDir.mkdtemp()
+        self.additional_path = self.mkdtemp()
         self.additional_git = Repo.init(self.additional_path)
         self.additional_dvc = DvcRepo.init(self.additional_path)
 

tests/func/test_ls.py

@@ -492,7 +492,7 @@ def test_ls_granular(erepo_dir):
     assert entries == [
         {"isout": True, "isdir": False, "isexec": False, "path": "1"},
         {"isout": True, "isdir": False, "isexec": False, "path": "2"},
-        {"isout": False, "isdir": True, "isexec": False, "path": "subdir"},
+        {"isout": True, "isdir": True, "isexec": False, "path": "subdir"},
     ]
 
 

tests/integration/plots/test_plots.py

@@ -44,7 +44,7 @@ def extract_vega_specs(html_path, plots_ids):
 
     result = {}
 
-    with open(html_path, "r", encoding="utf-8") as fd:
+    with open(html_path, encoding="utf-8") as fd:
         content = fd.read()
 
     reader = BeautifulSoup(content, features="html.parser")

tests/unit/repo/experiments/conftest.py

@@ -1,4 +1,4 @@
-from contextlib import contextmanager
+from functools import partial
 from textwrap import dedent
 
 import pytest
@@ -117,15 +117,13 @@ def failed_exp_stage(tmp_dir, scm, dvc):
     return stage
 
 
-@contextmanager
 def _thread_worker(app, **kwargs):
     # Based on pytest-celery's celery_worker fixture but using thread pool
     # instead of solo pool so that broadcast/control API is available
     from celery.contrib.testing import worker
 
     app.loader.import_task_module("celery.contrib.testing.tasks")
-    with worker.start_worker(app, pool="threads", **kwargs) as test_worker:
-        yield test_worker
+    return worker.start_worker(app, pool="threads", **kwargs)
 
 
 @pytest.fixture
@@ -134,12 +132,26 @@ def test_queue(tmp_dir, dvc, scm, mocker) -> LocalCeleryQueue:
 
     Test queue worker runs for the duration of the test in separate thread(s).
     """
-    celery_queue = dvc.experiments.celery_queue
-    mocker.patch.object(celery_queue, "spawn_worker")
-    with _thread_worker(
-        celery_queue.celery,
+    import celery
+
+    queue = dvc.experiments.celery_queue
+    mocker.patch.object(queue, "spawn_worker")
+
+    f = partial(
+        _thread_worker,
+        queue.celery,
         concurrency=1,
-        ping_task_timeout=30,
-    ) as worker:
-        mocker.patch.object(celery_queue, "worker", return_value=worker)
-        yield celery_queue
+        ping_task_timeout=20,
+    )
+    exc = None
+    for _ in range(3):
+        try:
+            with f() as worker:
+                mocker.patch.object(queue, "worker", return_value=worker)
+                yield queue
+                return
+        except celery.exceptions.TimeoutError as e:
+            exc = e
+            continue
+    assert exc
+    raise exc