diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c12911e6e..81d2bf1b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,15 +4,11 @@ repos: language_version: python3 repo: https://github.com/ambv/black rev: 19.10b0 - - repo: https://github.com/asottile/seed-isort-config - rev: v2.1.1 - hooks: - - id: seed-isort-config - hooks: - id: isort language_version: python3 repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.4.2 - hooks: - id: flake8 language_version: python3 diff --git a/.restyled.yaml b/.restyled.yaml index b8f0581fe..78d9bcc2d 100644 --- a/.restyled.yaml +++ b/.restyled.yaml @@ -10,7 +10,7 @@ restylers: interpreters: - python - name: isort - image: restyled/restyler-isort:v4.3.21 + image: restyled/restyler-isort:v5.4.2 command: - isort arguments: [] diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index daec0e8ce..bbed4e2e8 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -9,8 +9,9 @@ class CmdDaemonBase(CmdBaseNoRepo): class CmdDaemonUpdater(CmdDaemonBase): def run(self): import os - from dvc.repo import Repo + from dvc.config import Config + from dvc.repo import Repo from dvc.updater import Updater root_dir = Repo.find_root() diff --git a/dvc/command/dag.py b/dvc/command/dag.py index 27acb0b9f..2cf38e930 100644 --- a/dvc/command/dag.py +++ b/dvc/command/dag.py @@ -8,8 +8,8 @@ logger = logging.getLogger(__name__) def _show_ascii(G): - from dvc.repo.graph import get_pipelines from dvc.dagascii import draw + from dvc.repo.graph import get_pipelines pipelines = get_pipelines(G) @@ -22,6 +22,7 @@ def _show_ascii(G): def _show_dot(G): import io + from networkx.drawing.nx_pydot import write_dot dot_file = io.StringIO() @@ -31,6 +32,7 @@ def _show_dot(G): def _build(G, target=None, full=False): import networkx as nx + from dvc.repo.graph import get_pipeline, get_pipelines if target: diff --git a/dvc/command/experiments.py b/dvc/command/experiments.py index e74dcc53b..5af865f10 100644 --- a/dvc/command/experiments.py +++ b/dvc/command/experiments.py @@ -152,6 +152,7 @@ def _parse_list(param_list): def _show_experiments(all_experiments, console, precision=None, **kwargs): from rich.table import Table + from dvc.scm.git import Git include_metrics = _parse_list(kwargs.get("include_metrics", [])) @@ -193,6 +194,7 @@ def _show_experiments(all_experiments, console, precision=None, **kwargs): class CmdExperimentsShow(CmdBase): def run(self): from rich.console import Console + from dvc.utils.pager import pager if not self.repo.experiments: diff --git a/dvc/command/init.py b/dvc/command/init.py index bfd5bb88e..8f5e881fc 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -8,8 +8,8 @@ logger = logging.getLogger(__name__) class CmdInit(CmdBaseNoRepo): def run(self): - from dvc.repo import Repo from dvc.exceptions import InitError + from dvc.repo import Repo try: self.repo = Repo.init( diff --git a/dvc/command/metrics.py b/dvc/command/metrics.py index fcffba8fd..432f17345 100644 --- a/dvc/command/metrics.py +++ b/dvc/command/metrics.py @@ -15,6 +15,7 @@ def show_metrics( metrics, all_branches=False, all_tags=False, all_commits=False ): from flatten_json import flatten + from dvc.utils.diff import format_dict # When `metrics` contains a `None` key, it means that some files diff --git a/dvc/config.py b/dvc/config.py index e370489d6..685be624e 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -271,7 +271,7 @@ class Config(dict): @classmethod def get_dir(cls, level): - from appdirs import user_config_dir, site_config_dir + from appdirs import site_config_dir, user_config_dir assert level in ("global", "system") diff --git a/dvc/daemon.py b/dvc/daemon.py index 0251fb1f2..61eb8a7fd 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -25,7 +25,7 @@ def _popen(cmd, **kwargs): def _spawn_windows(cmd, env): - from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW + from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 329dafa03..c518b726f 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -42,43 +42,43 @@ def locked(f): class Repo: DVC_DIR = ".dvc" - from dvc.repo.destroy import destroy - from dvc.repo.install import install from dvc.repo.add import add - from dvc.repo.remove import remove - from dvc.repo.ls import ls - from dvc.repo.freeze import freeze, unfreeze - from dvc.repo.move import move - from dvc.repo.run import run - from dvc.repo.imp import imp - from dvc.repo.imp_url import imp_url - from dvc.repo.reproduce import reproduce - from dvc.repo.checkout import _checkout - from dvc.repo.push import push - from dvc.repo.fetch import _fetch - from dvc.repo.pull import pull - from dvc.repo.status import status - from dvc.repo.gc import gc - from dvc.repo.commit import commit - from dvc.repo.diff import diff from dvc.repo.brancher import brancher + from dvc.repo.checkout import _checkout + from dvc.repo.commit import commit + from dvc.repo.destroy import destroy + from dvc.repo.diff import diff + from dvc.repo.fetch import _fetch + from dvc.repo.freeze import freeze, unfreeze + from dvc.repo.gc import gc from dvc.repo.get import get from dvc.repo.get_url import get_url + from dvc.repo.imp import imp + from dvc.repo.imp_url import imp_url + from dvc.repo.install import install + from dvc.repo.ls import ls + from dvc.repo.move import move + from dvc.repo.pull import pull + from dvc.repo.push import push + from dvc.repo.remove import remove + from dvc.repo.reproduce import reproduce + from dvc.repo.run import run + from dvc.repo.status import status from dvc.repo.update import update def __init__(self, root_dir=None, scm=None, rev=None): - from dvc.state import State, StateNoop - from dvc.lock import make_lock - from dvc.scm import SCM from dvc.cache import Cache from dvc.data_cloud import DataCloud + from dvc.lock import make_lock from dvc.repo.experiments import Experiments from dvc.repo.metrics import Metrics - from dvc.repo.plots import Plots from dvc.repo.params import Params + from dvc.repo.plots import Plots + from dvc.scm import SCM + from dvc.stage.cache import StageCache + from dvc.state import State, StateNoop from dvc.tree.local import LocalTree from dvc.utils.fs import makedirs - from dvc.stage.cache import StageCache if scm: tree = scm.get_tree(rev) @@ -441,10 +441,11 @@ class Repo: """ import networkx as nx from pygtrie import Trie + from dvc.exceptions import ( OutputDuplicationError, - StagePathAsOutputError, OverlappingOutputPathsError, + StagePathAsOutputError, ) G = nx.DiGraph() diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index 20963e1fa..4f6ec5aec 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -404,6 +404,7 @@ class Experiments: def checkout_exp(self, rev): """Checkout an experiment to the user's workspace.""" from git.exc import GitCommandError + from dvc.repo.checkout import _checkout as dvc_checkout self._check_baseline(rev) diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index ea86ab4f8..44a64e29b 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -50,6 +50,7 @@ def gc( ) from contextlib import ExitStack + from dvc.repo import Repo if not repos: diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 1141b0077..f14011f52 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -20,8 +20,8 @@ class GetDVCFileError(DvcException): @staticmethod def get(url, path, out=None, rev=None): - from dvc.external_repo import external_repo from dvc.dvcfile import is_valid_filename + from dvc.external_repo import external_repo out = resolve_output(path, out) diff --git a/dvc/repo/graph.py b/dvc/repo/graph.py index fab001a48..325ea6d0b 100644 --- a/dvc/repo/graph.py +++ b/dvc/repo/graph.py @@ -1,5 +1,6 @@ def check_acyclic(graph): import networkx as nx + from dvc.exceptions import CyclicGraphError try: diff --git a/dvc/repo/move.py b/dvc/repo/move.py index 58f74b563..1036f7350 100644 --- a/dvc/repo/move.py +++ b/dvc/repo/move.py @@ -34,6 +34,7 @@ def move(self, from_path, to_path): """ import dvc.output as Output from dvc.stage import Stage + from ..dvcfile import DVC_FILE_SUFFIX, Dvcfile from_out = Output.loads_from(Stage(self), [from_path])[0] diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index 4386ca561..fd63ec228 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -210,7 +210,7 @@ def _prepare_plots(data, revs, props): def _render(datafile, datas, props, templates): - from .data import plot_data, PlotData + from .data import PlotData, plot_data # Copy it to not modify a passed value props = props.copy() diff --git a/dvc/repo/plots/data.py b/dvc/repo/plots/data.py index 91eed9abb..78d44164f 100644 --- a/dvc/repo/plots/data.py +++ b/dvc/repo/plots/data.py @@ -101,7 +101,7 @@ def _apply_path(data, path=None, **kwargs): def _lists(dictionary): for _, value in dictionary.items(): if isinstance(value, dict): - yield from (_lists(value)) + yield from _lists(value) elif isinstance(value, list): yield value diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 1f60117b4..6fc0cc6cd 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -120,7 +120,8 @@ def reproduce( def _parse_params(path_params): from flatten_json import unflatten - from yaml import safe_load, YAMLError + from yaml import YAMLError, safe_load + from dvc.dependency.param import ParamsDependency ret = {} diff --git a/dvc/repo/run.py b/dvc/repo/run.py index eea3eb05d..d383f4e25 100644 --- a/dvc/repo/run.py +++ b/dvc/repo/run.py @@ -36,7 +36,7 @@ def parse_params(path_params): def _get_file_path(kwargs): - from dvc.dvcfile import DVC_FILE_SUFFIX, DVC_FILE + from dvc.dvcfile import DVC_FILE, DVC_FILE_SUFFIX out = first( concat( @@ -76,8 +76,8 @@ def _check_stage_exists(dvcfile, stage): @locked @scm_context def run(self, fname=None, no_exec=False, single_stage=False, **kwargs): + from dvc.dvcfile import PIPELINE_FILE, Dvcfile from dvc.stage import Stage, create_stage - from dvc.dvcfile import Dvcfile, PIPELINE_FILE if not kwargs.get("cmd"): raise InvalidArgumentError("command is not specified") diff --git a/dvc/repo/tree.py b/dvc/repo/tree.py index a2762076f..09bcf286b 100644 --- a/dvc/repo/tree.py +++ b/dvc/repo/tree.py @@ -16,8 +16,8 @@ from dvc.utils.fs import copy_fobj_to_file, makedirs if TYPE_CHECKING: from dvc.repo import Repo - from dvc.tree.local import LocalTree from dvc.tree.git import GitTree + from dvc.tree.local import LocalTree logger = logging.getLogger(__name__) diff --git a/dvc/scm/git.py b/dvc/scm/git.py index 12b0b198c..5df270bda 100644 --- a/dvc/scm/git.py +++ b/dvc/scm/git.py @@ -393,9 +393,10 @@ class Git(Base): return self.repo.rev_parse("HEAD").hexsha def resolve_rev(self, rev): - from git.exc import BadName, GitCommandError from contextlib import suppress + from git.exc import BadName, GitCommandError + def _resolve_rev(name): with suppress(BadName, GitCommandError): try: diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 0802c036d..d77a78e3b 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -80,7 +80,7 @@ class StageCache: return None def _create_stage(self, cache, wdir=None): - from . import create_stage, PipelineStage + from . import PipelineStage, create_stage stage = create_stage( PipelineStage, diff --git a/dvc/stage/decorators.py b/dvc/stage/decorators.py index 3623fb525..df51f67f8 100644 --- a/dvc/stage/decorators.py +++ b/dvc/stage/decorators.py @@ -6,8 +6,9 @@ from funcy import decorator @decorator def rwlocked(call, read=None, write=None): import sys - from dvc.rwlock import rwlock + from dvc.dependency.repo import RepoDependency + from dvc.rwlock import rwlock if read is None: read = [] diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 5957277d9..feb2a348f 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -110,9 +110,10 @@ def check_circular_dependency(stage): def check_duplicated_arguments(stage): - from dvc.exceptions import ArgumentDuplicationError from collections import Counter + from dvc.exceptions import ArgumentDuplicationError + path_counts = Counter(edge.path_info for edge in stage.deps + stage.outs) for path, occurrence in path_counts.items(): diff --git a/dvc/system.py b/dvc/system.py index 3439f4119..1d7f0a9b5 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -119,13 +119,14 @@ class System: @staticmethod def _getdirinfo(path): from collections import namedtuple + from win32file import ( # pylint: disable=import-error - CreateFileW, - GetFileInformationByHandle, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_READ, OPEN_EXISTING, + CreateFileW, + GetFileInformationByHandle, ) # NOTE: use FILE_FLAG_OPEN_REPARSE_POINT to open symlink itself and not diff --git a/dvc/tree/azure.py b/dvc/tree/azure.py index 5ef3f2594..ee094360d 100644 --- a/dvc/tree/azure.py +++ b/dvc/tree/azure.py @@ -66,8 +66,8 @@ class AzureTree(BaseTree): @cached_property def blob_service(self): # pylint: disable=no-name-in-module - from azure.storage.blob import BlobServiceClient from azure.core.exceptions import ResourceNotFoundError + from azure.storage.blob import BlobServiceClient logger.debug(f"URL {self.path_info}") diff --git a/dvc/tree/gs.py b/dvc/tree/gs.py index c7055a73a..c35a490b4 100644 --- a/dvc/tree/gs.py +++ b/dvc/tree/gs.py @@ -28,7 +28,6 @@ def dynamic_chunk_size(func): # Note: must be multiple of 256K. # # [#2572]: https://github.com/iterative/dvc/issues/2572 - # skipcq: PYL-W0212 multiplier = 40 while True: diff --git a/dvc/updater.py b/dvc/updater.py index 92ef1bb88..1eda2a4c8 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -88,6 +88,7 @@ class Updater: # pragma: no cover def _get_latest_version(self): import json + import requests try: diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 8fece228f..bcceb1f76 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -45,8 +45,8 @@ def _fobj_md5(fobj, hash_md5, binary, progress_func=None): def file_md5(fname, tree=None): """ get the (md5 hexdigest, md5 digest) of a file """ - from dvc.progress import Tqdm from dvc.istextfile import istextfile + from dvc.progress import Tqdm if tree: exists_func = tree.exists @@ -364,6 +364,7 @@ def resolve_output(inp, out): def resolve_paths(repo, out): from urllib.parse import urlparse + from ..dvcfile import DVC_FILE_SUFFIX from ..path_info import PathInfo from .fs import contains_symlink_up_to diff --git a/setup.cfg b/setup.cfg index c794a46e1..0c5df19e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,10 +15,6 @@ show_source=true count=true [isort] -include_trailing_comma=true +profile=black known_first_party=dvc,tests -known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,toml,tqdm,voluptuous,yaml,zc line_length=79 -force_grid_wrap=0 -use_parentheses=True -multi_line_output=3 diff --git a/tests/__init__.py b/tests/__init__.py index 8d69f84bf..343dc6fa0 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,9 +2,10 @@ import os # Increasing fd ulimit for tests if os.name == "nt": - import win32file # pylint: disable=import-error import subprocess + import win32file # pylint: disable=import-error + win32file._setmaxstdio(2048) # Workaround for two bugs: diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 92f9525e9..b712d744c 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -595,8 +595,9 @@ class TestAddUnprotected(TestDvc): @pytest.fixture def temporary_windows_drive(tmp_path_factory): import string - import win32api # pylint: disable=import-error from ctypes import windll + + import win32api # pylint: disable=import-error from win32con import DDD_REMOVE_DEFINITION # pylint: disable=import-error drives = [ diff --git a/tests/func/test_ls.py b/tests/func/test_ls.py index 4c2681d52..0f65c52f4 100644 --- a/tests/func/test_ls.py +++ b/tests/func/test_ls.py @@ -74,6 +74,7 @@ def test_ls_repo(tmp_dir, dvc, scm): def test_ls_repo_with_color(tmp_dir, dvc, scm, mocker, monkeypatch, caplog): import logging + from dvc.cli import parse_args tmp_dir.scm_gen(FS_STRUCTURE, commit="init") diff --git a/tests/func/test_repro_multistage.py b/tests/func/test_repro_multistage.py index 1dd363a7d..c6875f270 100644 --- a/tests/func/test_repro_multistage.py +++ b/tests/func/test_repro_multistage.py @@ -471,9 +471,8 @@ def test_cyclic_graph_error(tmp_dir, dvc, run_copy): def test_repro_multiple_params(tmp_dir, dvc): - from tests.func.test_run_multistage import supported_params - from dvc.stage.utils import split_params_deps + from tests.func.test_run_multistage import supported_params with (tmp_dir / "params2.yaml").open("w+") as f: yaml.dump(supported_params, f) diff --git a/tests/func/test_run_multistage.py b/tests/func/test_run_multistage.py index 056052342..8be4c3a04 100644 --- a/tests/func/test_run_multistage.py +++ b/tests/func/test_run_multistage.py @@ -11,8 +11,8 @@ from dvc.utils.yaml import parse_yaml_for_update def test_run_with_name(tmp_dir, dvc, run_copy): - from dvc.stage import PipelineStage from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK + from dvc.stage import PipelineStage tmp_dir.dvc_gen("foo", "foo") assert not os.path.exists(PIPELINE_FILE) @@ -24,8 +24,8 @@ def test_run_with_name(tmp_dir, dvc, run_copy): def test_run_no_exec(tmp_dir, dvc, run_copy): - from dvc.stage import PipelineStage from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK + from dvc.stage import PipelineStage tmp_dir.dvc_gen("foo", "foo") assert not os.path.exists(PIPELINE_FILE) @@ -62,8 +62,8 @@ def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy): def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy): + from dvc.dvcfile import PIPELINE_FILE, Dvcfile from dvc.stage import PipelineStage - from dvc.dvcfile import Dvcfile, PIPELINE_FILE tmp_dir.dvc_gen("foo", "foo") run_copy("foo", "foo1", name="copy-foo-foo1") @@ -144,7 +144,7 @@ def test_graph(tmp_dir, dvc): def test_run_dump_on_multistage(tmp_dir, dvc, run_head): - from dvc.dvcfile import Dvcfile, PIPELINE_FILE + from dvc.dvcfile import PIPELINE_FILE, Dvcfile tmp_dir.gen( { diff --git a/tests/remotes/azure.py b/tests/remotes/azure.py index 27db33298..b0b31b1d2 100644 --- a/tests/remotes/azure.py +++ b/tests/remotes/azure.py @@ -24,12 +24,12 @@ class Azure(Base, CloudURLInfo): @pytest.fixture(scope="session") def azure_server(docker_compose, docker_services): - from azure.storage.blob import ( # pylint: disable=no-name-in-module - BlobServiceClient, - ) from azure.core.exceptions import ( # pylint: disable=no-name-in-module AzureError, ) + from azure.storage.blob import ( # pylint: disable=no-name-in-module + BlobServiceClient, + ) port = docker_services.port_for("azurite", 10000) connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port) diff --git a/tests/remotes/hdfs.py b/tests/remotes/hdfs.py index f7502b797..9b5601b8d 100644 --- a/tests/remotes/hdfs.py +++ b/tests/remotes/hdfs.py @@ -70,8 +70,9 @@ class HDFS(Base, URLInfo): # pylint: disable=abstract-method @pytest.fixture(scope="session") def hadoop(): - import wget import tarfile + + import wget from appdirs import user_cache_dir if platform.system() != "Linux": diff --git a/tests/unit/scm/test_git.py b/tests/unit/scm/test_git.py index 04b630b6a..2f82dd04e 100644 --- a/tests/unit/scm/test_git.py +++ b/tests/unit/scm/test_git.py @@ -98,8 +98,8 @@ def test_is_tracked_unicode(tmp_dir, scm): def test_no_commits(tmp_dir): - from tests.dir_helpers import git_init from dvc.scm.git import Git + from tests.dir_helpers import git_init git_init(".") assert Git().no_commits diff --git a/tests/utils/httpd.py b/tests/utils/httpd.py index 4db94adf1..74ddbc2b7 100644 --- a/tests/utils/httpd.py +++ b/tests/utils/httpd.py @@ -16,8 +16,8 @@ class TestRequestHandler(RangeRequestHandler): super().__init__(*args, **kwargs) def translate_path(self, path): - import urllib import posixpath + import urllib # NOTE: `directory` was introduced in 3.7 if sys.version_info >= (3, 7):