upgrade to isort5 (#4399)

* Imports are no longer moved to top.
* `seed-isort-config` is no longer needed.
* Has a `black`-compatible profile/config.
* Also `isorts` the imports anywhere in the code
  (not just at the top).

Also bumps `restyled-isort` to newest version

.pre-commit-config.yaml

@@ -4,15 +4,11 @@ repos:
         language_version: python3
     repo: https://github.com/ambv/black
     rev: 19.10b0
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.1.1
-    hooks:
-      - id: seed-isort-config
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 4.3.21
+    rev: 5.4.2
   - hooks:
       - id: flake8
         language_version: python3

.restyled.yaml

@@ -10,7 +10,7 @@ restylers:
     interpreters:
     - python
   - name: isort
-    image: restyled/restyler-isort:v4.3.21
+    image: restyled/restyler-isort:v5.4.2
     command:
     - isort
     arguments: []

dvc/command/daemon.py

@@ -9,8 +9,9 @@ class CmdDaemonBase(CmdBaseNoRepo):
 class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
-        from dvc.repo import Repo
+
         from dvc.config import Config
+        from dvc.repo import Repo
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()

dvc/command/dag.py

@@ -8,8 +8,8 @@ logger = logging.getLogger(__name__)
 
 
 def _show_ascii(G):
-    from dvc.repo.graph import get_pipelines
     from dvc.dagascii import draw
+    from dvc.repo.graph import get_pipelines
 
     pipelines = get_pipelines(G)
 
@@ -22,6 +22,7 @@ def _show_ascii(G):
 
 def _show_dot(G):
     import io
+
     from networkx.drawing.nx_pydot import write_dot
 
     dot_file = io.StringIO()
@@ -31,6 +32,7 @@ def _show_dot(G):
 
 def _build(G, target=None, full=False):
     import networkx as nx
+
     from dvc.repo.graph import get_pipeline, get_pipelines
 
     if target:

dvc/command/experiments.py

@@ -152,6 +152,7 @@ def _parse_list(param_list):
 
 def _show_experiments(all_experiments, console, precision=None, **kwargs):
     from rich.table import Table
+
     from dvc.scm.git import Git
 
     include_metrics = _parse_list(kwargs.get("include_metrics", []))
@@ -193,6 +194,7 @@ def _show_experiments(all_experiments, console, precision=None, **kwargs):
 class CmdExperimentsShow(CmdBase):
     def run(self):
         from rich.console import Console
+
         from dvc.utils.pager import pager
 
         if not self.repo.experiments:

dvc/command/init.py

@@ -8,8 +8,8 @@ logger = logging.getLogger(__name__)
 
 class CmdInit(CmdBaseNoRepo):
     def run(self):
-        from dvc.repo import Repo
         from dvc.exceptions import InitError
+        from dvc.repo import Repo
 
         try:
             self.repo = Repo.init(

dvc/command/metrics.py

@@ -15,6 +15,7 @@ def show_metrics(
     metrics, all_branches=False, all_tags=False, all_commits=False
 ):
     from flatten_json import flatten
+
     from dvc.utils.diff import format_dict
 
     # When `metrics` contains a `None` key, it means that some files

dvc/config.py

@@ -271,7 +271,7 @@ class Config(dict):
 
     @classmethod
     def get_dir(cls, level):
-        from appdirs import user_config_dir, site_config_dir
+        from appdirs import site_config_dir, user_config_dir
 
         assert level in ("global", "system")
 

dvc/daemon.py

@@ -25,7 +25,7 @@ def _popen(cmd, **kwargs):
 
 
 def _spawn_windows(cmd, env):
-    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
 
     creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 

dvc/repo/__init__.py

@@ -42,43 +42,43 @@ def locked(f):
 class Repo:
     DVC_DIR = ".dvc"
 
-    from dvc.repo.destroy import destroy
-    from dvc.repo.install import install
     from dvc.repo.add import add
-    from dvc.repo.remove import remove
-    from dvc.repo.ls import ls
-    from dvc.repo.freeze import freeze, unfreeze
-    from dvc.repo.move import move
-    from dvc.repo.run import run
-    from dvc.repo.imp import imp
-    from dvc.repo.imp_url import imp_url
-    from dvc.repo.reproduce import reproduce
-    from dvc.repo.checkout import _checkout
-    from dvc.repo.push import push
-    from dvc.repo.fetch import _fetch
-    from dvc.repo.pull import pull
-    from dvc.repo.status import status
-    from dvc.repo.gc import gc
-    from dvc.repo.commit import commit
-    from dvc.repo.diff import diff
     from dvc.repo.brancher import brancher
+    from dvc.repo.checkout import _checkout
+    from dvc.repo.commit import commit
+    from dvc.repo.destroy import destroy
+    from dvc.repo.diff import diff
+    from dvc.repo.fetch import _fetch
+    from dvc.repo.freeze import freeze, unfreeze
+    from dvc.repo.gc import gc
     from dvc.repo.get import get
     from dvc.repo.get_url import get_url
+    from dvc.repo.imp import imp
+    from dvc.repo.imp_url import imp_url
+    from dvc.repo.install import install
+    from dvc.repo.ls import ls
+    from dvc.repo.move import move
+    from dvc.repo.pull import pull
+    from dvc.repo.push import push
+    from dvc.repo.remove import remove
+    from dvc.repo.reproduce import reproduce
+    from dvc.repo.run import run
+    from dvc.repo.status import status
     from dvc.repo.update import update
 
     def __init__(self, root_dir=None, scm=None, rev=None):
-        from dvc.state import State, StateNoop
-        from dvc.lock import make_lock
-        from dvc.scm import SCM
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
+        from dvc.lock import make_lock
         from dvc.repo.experiments import Experiments
         from dvc.repo.metrics import Metrics
-        from dvc.repo.plots import Plots
         from dvc.repo.params import Params
+        from dvc.repo.plots import Plots
+        from dvc.scm import SCM
+        from dvc.stage.cache import StageCache
+        from dvc.state import State, StateNoop
         from dvc.tree.local import LocalTree
         from dvc.utils.fs import makedirs
-        from dvc.stage.cache import StageCache
 
         if scm:
             tree = scm.get_tree(rev)
@@ -441,10 +441,11 @@ class Repo:
         """
         import networkx as nx
         from pygtrie import Trie
+
         from dvc.exceptions import (
             OutputDuplicationError,
-            StagePathAsOutputError,
             OverlappingOutputPathsError,
+            StagePathAsOutputError,
         )
 
         G = nx.DiGraph()

dvc/repo/experiments/__init__.py

@@ -404,6 +404,7 @@ class Experiments:
     def checkout_exp(self, rev):
         """Checkout an experiment to the user's workspace."""
         from git.exc import GitCommandError
+
         from dvc.repo.checkout import _checkout as dvc_checkout
 
         self._check_baseline(rev)

dvc/repo/gc.py

@@ -50,6 +50,7 @@ def gc(
     )
 
     from contextlib import ExitStack
+
     from dvc.repo import Repo
 
     if not repos:

dvc/repo/get.py

@@ -20,8 +20,8 @@ class GetDVCFileError(DvcException):
 
 @staticmethod
 def get(url, path, out=None, rev=None):
-    from dvc.external_repo import external_repo
     from dvc.dvcfile import is_valid_filename
+    from dvc.external_repo import external_repo
 
     out = resolve_output(path, out)
 

dvc/repo/graph.py

@@ -1,5 +1,6 @@
 def check_acyclic(graph):
     import networkx as nx
+
     from dvc.exceptions import CyclicGraphError
 
     try:

dvc/repo/move.py

@@ -34,6 +34,7 @@ def move(self, from_path, to_path):
     """
     import dvc.output as Output
     from dvc.stage import Stage
+
     from ..dvcfile import DVC_FILE_SUFFIX, Dvcfile
 
     from_out = Output.loads_from(Stage(self), [from_path])[0]

dvc/repo/plots/__init__.py

@@ -210,7 +210,7 @@ def _prepare_plots(data, revs, props):
 
 
 def _render(datafile, datas, props, templates):
-    from .data import plot_data, PlotData
+    from .data import PlotData, plot_data
 
     # Copy it to not modify a passed value
     props = props.copy()

dvc/repo/plots/data.py

@@ -101,7 +101,7 @@ def _apply_path(data, path=None, **kwargs):
 def _lists(dictionary):
     for _, value in dictionary.items():
         if isinstance(value, dict):
-            yield from (_lists(value))
+            yield from _lists(value)
         elif isinstance(value, list):
             yield value
 

dvc/repo/reproduce.py

@@ -120,7 +120,8 @@ def reproduce(
 
 def _parse_params(path_params):
     from flatten_json import unflatten
-    from yaml import safe_load, YAMLError
+    from yaml import YAMLError, safe_load
+
     from dvc.dependency.param import ParamsDependency
 
     ret = {}

dvc/repo/run.py

@@ -36,7 +36,7 @@ def parse_params(path_params):
 
 
 def _get_file_path(kwargs):
-    from dvc.dvcfile import DVC_FILE_SUFFIX, DVC_FILE
+    from dvc.dvcfile import DVC_FILE, DVC_FILE_SUFFIX
 
     out = first(
         concat(
@@ -76,8 +76,8 @@ def _check_stage_exists(dvcfile, stage):
 @locked
 @scm_context
 def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
+    from dvc.dvcfile import PIPELINE_FILE, Dvcfile
     from dvc.stage import Stage, create_stage
-    from dvc.dvcfile import Dvcfile, PIPELINE_FILE
 
     if not kwargs.get("cmd"):
         raise InvalidArgumentError("command is not specified")

dvc/repo/tree.py

@@ -16,8 +16,8 @@ from dvc.utils.fs import copy_fobj_to_file, makedirs
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
-    from dvc.tree.local import LocalTree
     from dvc.tree.git import GitTree
+    from dvc.tree.local import LocalTree
 
 
 logger = logging.getLogger(__name__)

dvc/scm/git.py

@@ -393,9 +393,10 @@ class Git(Base):
         return self.repo.rev_parse("HEAD").hexsha
 
     def resolve_rev(self, rev):
-        from git.exc import BadName, GitCommandError
         from contextlib import suppress
 
+        from git.exc import BadName, GitCommandError
+
         def _resolve_rev(name):
             with suppress(BadName, GitCommandError):
                 try:

dvc/stage/cache.py

@@ -80,7 +80,7 @@ class StageCache:
         return None
 
     def _create_stage(self, cache, wdir=None):
-        from . import create_stage, PipelineStage
+        from . import PipelineStage, create_stage
 
         stage = create_stage(
             PipelineStage,

dvc/stage/decorators.py

@@ -6,8 +6,9 @@ from funcy import decorator
 @decorator
 def rwlocked(call, read=None, write=None):
     import sys
-    from dvc.rwlock import rwlock
+
     from dvc.dependency.repo import RepoDependency
+    from dvc.rwlock import rwlock
 
     if read is None:
         read = []

dvc/stage/utils.py

@@ -110,9 +110,10 @@ def check_circular_dependency(stage):
 
 
 def check_duplicated_arguments(stage):
-    from dvc.exceptions import ArgumentDuplicationError
     from collections import Counter
 
+    from dvc.exceptions import ArgumentDuplicationError
+
     path_counts = Counter(edge.path_info for edge in stage.deps + stage.outs)
 
     for path, occurrence in path_counts.items():

dvc/system.py

@@ -119,13 +119,14 @@ class System:
     @staticmethod
     def _getdirinfo(path):
         from collections import namedtuple
+
         from win32file import (  # pylint: disable=import-error
-            CreateFileW,
-            GetFileInformationByHandle,
             FILE_FLAG_BACKUP_SEMANTICS,
             FILE_FLAG_OPEN_REPARSE_POINT,
             FILE_SHARE_READ,
             OPEN_EXISTING,
+            CreateFileW,
+            GetFileInformationByHandle,
         )
 
         # NOTE: use FILE_FLAG_OPEN_REPARSE_POINT to open symlink itself and not

dvc/tree/azure.py

@@ -66,8 +66,8 @@ class AzureTree(BaseTree):
     @cached_property
     def blob_service(self):
         # pylint: disable=no-name-in-module
-        from azure.storage.blob import BlobServiceClient
         from azure.core.exceptions import ResourceNotFoundError
+        from azure.storage.blob import BlobServiceClient
 
         logger.debug(f"URL {self.path_info}")
 

dvc/tree/gs.py

@@ -28,7 +28,6 @@ def dynamic_chunk_size(func):
         # Note: must be multiple of 256K.
         #
         # [#2572]: https://github.com/iterative/dvc/issues/2572
-
         # skipcq: PYL-W0212
         multiplier = 40
         while True:

dvc/updater.py

@@ -88,6 +88,7 @@ class Updater:  # pragma: no cover
 
     def _get_latest_version(self):
         import json
+
         import requests
 
         try:

dvc/utils/__init__.py

@@ -45,8 +45,8 @@ def _fobj_md5(fobj, hash_md5, binary, progress_func=None):
 
 def file_md5(fname, tree=None):
     """ get the (md5 hexdigest, md5 digest) of a file """
-    from dvc.progress import Tqdm
     from dvc.istextfile import istextfile
+    from dvc.progress import Tqdm
 
     if tree:
         exists_func = tree.exists
@@ -364,6 +364,7 @@ def resolve_output(inp, out):
 
 def resolve_paths(repo, out):
     from urllib.parse import urlparse
+
     from ..dvcfile import DVC_FILE_SUFFIX
     from ..path_info import PathInfo
     from .fs import contains_symlink_up_to

setup.cfg

@@ -15,10 +15,6 @@ show_source=true
 count=true
 
 [isort]
-include_trailing_comma=true
+profile=black
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,toml,tqdm,voluptuous,yaml,zc
 line_length=79
-force_grid_wrap=0
-use_parentheses=True
-multi_line_output=3

tests/__init__.py

@@ -2,9 +2,10 @@ import os
 
 # Increasing fd ulimit for tests
 if os.name == "nt":
-    import win32file  # pylint: disable=import-error
     import subprocess
 
+    import win32file  # pylint: disable=import-error
+
     win32file._setmaxstdio(2048)
 
     # Workaround for two bugs:

tests/func/test_add.py

@@ -595,8 +595,9 @@ class TestAddUnprotected(TestDvc):
 @pytest.fixture
 def temporary_windows_drive(tmp_path_factory):
     import string
-    import win32api  # pylint: disable=import-error
     from ctypes import windll
+
+    import win32api  # pylint: disable=import-error
     from win32con import DDD_REMOVE_DEFINITION  # pylint: disable=import-error
 
     drives = [

tests/func/test_ls.py

@@ -74,6 +74,7 @@ def test_ls_repo(tmp_dir, dvc, scm):
 
 def test_ls_repo_with_color(tmp_dir, dvc, scm, mocker, monkeypatch, caplog):
     import logging
+
     from dvc.cli import parse_args
 
     tmp_dir.scm_gen(FS_STRUCTURE, commit="init")

tests/func/test_repro_multistage.py

@@ -471,9 +471,8 @@ def test_cyclic_graph_error(tmp_dir, dvc, run_copy):
 
 
 def test_repro_multiple_params(tmp_dir, dvc):
-    from tests.func.test_run_multistage import supported_params
-
     from dvc.stage.utils import split_params_deps
+    from tests.func.test_run_multistage import supported_params
 
     with (tmp_dir / "params2.yaml").open("w+") as f:
         yaml.dump(supported_params, f)

tests/func/test_run_multistage.py

@@ -11,8 +11,8 @@ from dvc.utils.yaml import parse_yaml_for_update
 
 
 def test_run_with_name(tmp_dir, dvc, run_copy):
-    from dvc.stage import PipelineStage
     from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
+    from dvc.stage import PipelineStage
 
     tmp_dir.dvc_gen("foo", "foo")
     assert not os.path.exists(PIPELINE_FILE)
@@ -24,8 +24,8 @@ def test_run_with_name(tmp_dir, dvc, run_copy):
 
 
 def test_run_no_exec(tmp_dir, dvc, run_copy):
-    from dvc.stage import PipelineStage
     from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
+    from dvc.stage import PipelineStage
 
     tmp_dir.dvc_gen("foo", "foo")
     assert not os.path.exists(PIPELINE_FILE)
@@ -62,8 +62,8 @@ def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy):
 
 
 def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy):
+    from dvc.dvcfile import PIPELINE_FILE, Dvcfile
     from dvc.stage import PipelineStage
-    from dvc.dvcfile import Dvcfile, PIPELINE_FILE
 
     tmp_dir.dvc_gen("foo", "foo")
     run_copy("foo", "foo1", name="copy-foo-foo1")
@@ -144,7 +144,7 @@ def test_graph(tmp_dir, dvc):
 
 
 def test_run_dump_on_multistage(tmp_dir, dvc, run_head):
-    from dvc.dvcfile import Dvcfile, PIPELINE_FILE
+    from dvc.dvcfile import PIPELINE_FILE, Dvcfile
 
     tmp_dir.gen(
         {

tests/remotes/azure.py

@@ -24,12 +24,12 @@ class Azure(Base, CloudURLInfo):
 
 @pytest.fixture(scope="session")
 def azure_server(docker_compose, docker_services):
-    from azure.storage.blob import (  # pylint: disable=no-name-in-module
-        BlobServiceClient,
-    )
     from azure.core.exceptions import (  # pylint: disable=no-name-in-module
         AzureError,
     )
+    from azure.storage.blob import (  # pylint: disable=no-name-in-module
+        BlobServiceClient,
+    )
 
     port = docker_services.port_for("azurite", 10000)
     connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port)

tests/remotes/hdfs.py

@@ -70,8 +70,9 @@ class HDFS(Base, URLInfo):  # pylint: disable=abstract-method
 
 @pytest.fixture(scope="session")
 def hadoop():
-    import wget
     import tarfile
+
+    import wget
     from appdirs import user_cache_dir
 
     if platform.system() != "Linux":

tests/unit/scm/test_git.py

@@ -98,8 +98,8 @@ def test_is_tracked_unicode(tmp_dir, scm):
 
 
 def test_no_commits(tmp_dir):
-    from tests.dir_helpers import git_init
     from dvc.scm.git import Git
+    from tests.dir_helpers import git_init
 
     git_init(".")
     assert Git().no_commits

tests/utils/httpd.py

@@ -16,8 +16,8 @@ class TestRequestHandler(RangeRequestHandler):
         super().__init__(*args, **kwargs)
 
     def translate_path(self, path):
-        import urllib
         import posixpath
+        import urllib
 
         # NOTE: `directory` was introduced in 3.7
         if sys.version_info >= (3, 7):