From 27f581cf4e4557b0cb19b58ecf801c0309475ce4 Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Tue, 31 Dec 2019 23:15:27 -0600 Subject: [PATCH] py3: remove imports from utils.compat --- dvc/analytics.py | 1 - dvc/api.py | 10 ++-------- dvc/command/version.py | 2 +- dvc/config.py | 1 - dvc/dependency/__init__.py | 4 ++-- dvc/dependency/repo.py | 1 - dvc/istextfile.py | 2 -- dvc/output/__init__.py | 2 +- dvc/output/base.py | 2 +- dvc/output/local.py | 2 +- dvc/path_info.py | 4 ++-- dvc/prompt.py | 3 --- dvc/remote/azure.py | 2 +- dvc/remote/base.py | 4 +--- dvc/remote/config.py | 2 +- dvc/remote/gs.py | 1 - dvc/remote/hdfs.py | 4 ++-- dvc/remote/http.py | 1 - dvc/remote/local.py | 1 - dvc/remote/ssh/__init__.py | 2 +- dvc/repo/__init__.py | 6 +++--- dvc/repo/gc.py | 2 +- dvc/repo/get.py | 1 - dvc/repo/metrics/show.py | 1 - dvc/rwlock.py | 3 +-- dvc/scm/git/__init__.py | 1 - dvc/scm/tree.py | 2 +- dvc/stage.py | 2 +- dvc/state.py | 3 +-- dvc/system.py | 1 - dvc/utils/__init__.py | 4 +--- dvc/utils/collections.py | 2 +- dvc/utils/http.py | 2 -- dvc/utils/stage.py | 1 - scripts/innosetup/config_gen.py | 7 ++++--- tests/basic_env.py | 1 - tests/dir_helpers.py | 3 ++- tests/func/test_add.py | 2 +- tests/func/test_repro.py | 4 ++-- tests/unit/test_path_info.py | 2 +- tests/utils/httpd.py | 9 +-------- 41 files changed, 37 insertions(+), 73 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index d843d04fa..18aea6428 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -17,7 +17,6 @@ from dvc.lock import Lock, LockError from dvc.repo import Repo from dvc.scm import SCM from dvc.utils import env2bool, is_binary, makedirs -from dvc.utils.compat import FileNotFoundError logger = logging.getLogger(__name__) diff --git a/dvc/api.py b/dvc/api.py index 447cf60cd..b6a6b2cdd 100644 --- a/dvc/api.py +++ b/dvc/api.py @@ -2,14 +2,8 @@ import importlib import os import sys import copy -from contextlib import contextmanager - -try: - from contextlib import _GeneratorContextManager as GCM -except ImportError: - from contextlib import GeneratorContextManager as GCM - -from dvc.utils.compat import urlparse +from urllib.parse import urlparse +from contextlib import contextmanager, _GeneratorContextManager as GCM import ruamel.yaml from voluptuous import Schema, Required, Invalid diff --git a/dvc/command/version.py b/dvc/command/version.py index 301dd6554..aa7d630af 100644 --- a/dvc/command/version.py +++ b/dvc/command/version.py @@ -4,6 +4,7 @@ import logging import os import platform import uuid +import pathlib try: import psutil @@ -11,7 +12,6 @@ except ImportError: psutil = None from dvc.utils import is_binary, relpath -from dvc.utils.compat import pathlib from dvc.command.base import CmdBaseNoRepo, append_doc_link from dvc.version import __version__ from dvc.exceptions import DvcException, NotDvcRepoError diff --git a/dvc/config.py b/dvc/config.py index e9fc0d598..206e17317 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -12,7 +12,6 @@ from voluptuous import All, Any, Lower, Range, Coerce, Match from dvc.exceptions import DvcException from dvc.exceptions import NotDvcRepoError -from dvc.utils.compat import open logger = logging.getLogger(__name__) diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index a963ab380..a968f0144 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -1,3 +1,5 @@ +from urllib.parse import urlparse + import dvc.output as output from dvc.dependency.gs import DependencyGS from dvc.dependency.hdfs import DependencyHDFS @@ -44,8 +46,6 @@ SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA def _get(stage, p, info): - from dvc.utils.compat import urlparse - parsed = urlparse(p) if parsed.scheme == "remote": diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 0963e1061..2bcda628e 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -1,7 +1,6 @@ import copy import os from contextlib import contextmanager -from dvc.utils.compat import FileNotFoundError from funcy import merge diff --git a/dvc/istextfile.py b/dvc/istextfile.py index 47c1a1494..2e10592d9 100644 --- a/dvc/istextfile.py +++ b/dvc/istextfile.py @@ -1,7 +1,5 @@ """Use heuristics to guess if it is a text file or a binary file.""" -from dvc.utils.compat import open - # Based on https://eli.thegreenplace.net/2011/10/19/ # perls-guess-if-file-is-text-or-binary-implemented-in-python diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 075b623f9..abcf68191 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -1,3 +1,4 @@ +from urllib.parse import urlparse from voluptuous import Any, Required from dvc.output.base import OutputBase @@ -11,7 +12,6 @@ from dvc.remote.hdfs import RemoteHDFS from dvc.remote.local import RemoteLOCAL from dvc.remote.s3 import RemoteS3 from dvc.scheme import Schemes -from dvc.utils.compat import urlparse OUTS = [ OutputHDFS, diff --git a/dvc/output/base.py b/dvc/output/base.py index 8c7d99b7c..edc5780bd 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -1,4 +1,5 @@ import logging +from urllib.parse import urlparse from copy import copy from voluptuous import Any @@ -8,7 +9,6 @@ from dvc.cache import NamedCache from dvc.exceptions import CollectCacheError from dvc.exceptions import DvcException from dvc.remote.base import RemoteBASE -from dvc.utils.compat import urlparse logger = logging.getLogger(__name__) diff --git a/dvc/output/local.py b/dvc/output/local.py index 8e1de6df2..64e2e3388 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -1,5 +1,6 @@ import logging import os +from urllib.parse import urlparse from dvc.exceptions import DvcException from dvc.istextfile import istextfile @@ -7,7 +8,6 @@ from dvc.output.base import OutputBase from dvc.remote.local import RemoteLOCAL from dvc.utils import relpath from dvc.utils.compat import fspath_py35 -from dvc.utils.compat import urlparse from dvc.utils.fs import path_isin diff --git a/dvc/path_info.py b/dvc/path_info.py index dc64bc7a1..6b4c300b5 100644 --- a/dvc/path_info.py +++ b/dvc/path_info.py @@ -1,11 +1,11 @@ import os import posixpath +import pathlib +from urllib.parse import urlparse from funcy import cached_property from dvc.utils import relpath -from dvc.utils.compat import pathlib -from dvc.utils.compat import urlparse class _BasePath(object): diff --git a/dvc/prompt.py b/dvc/prompt.py index 7711dc5fe..e71aa6e67 100644 --- a/dvc/prompt.py +++ b/dvc/prompt.py @@ -4,9 +4,6 @@ import logging import sys from getpass import getpass -from dvc.utils.compat import input - - logger = logging.getLogger(__name__) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index f80586ff6..96967aae4 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -2,6 +2,7 @@ import logging import os import re from datetime import datetime, timedelta +from urllib.parse import urlparse import threading from funcy import cached_property, wrap_prop @@ -11,7 +12,6 @@ from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.remote.base import RemoteBASE from dvc.scheme import Schemes -from dvc.utils.compat import urlparse logger = logging.getLogger(__name__) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index b215a5586..e9ec778b6 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -1,11 +1,9 @@ import errno - -from dvc.utils.compat import FileNotFoundError, urlparse - import itertools import json import logging import tempfile +from urllib.parse import urlparse from concurrent.futures import ThreadPoolExecutor from copy import copy from functools import partial diff --git a/dvc/remote/config.py b/dvc/remote/config.py index bbda7fc44..87572bd1d 100644 --- a/dvc/remote/config.py +++ b/dvc/remote/config.py @@ -1,11 +1,11 @@ import logging import os import posixpath +from urllib.parse import urlparse from dvc.config import Config from dvc.config import ConfigError from dvc.utils import relpath -from dvc.utils.compat import urlparse logger = logging.getLogger(__name__) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 547180c3c..05c620fb0 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -13,7 +13,6 @@ from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.remote.base import RemoteBASE from dvc.scheme import Schemes -from dvc.utils.compat import FileNotFoundError # skipcq: PYL-W0622 logger = logging.getLogger(__name__) diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 49579003a..d6adeb02e 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -1,12 +1,12 @@ -from dvc.utils.compat import FileNotFoundError, urlparse import io import logging import os import posixpath import re +import subprocess from collections import deque from contextlib import closing, contextmanager -import subprocess +from urllib.parse import urlparse from .base import RemoteBASE, RemoteCmdError from .pool import get_connection diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 452f26c76..9dbb6fa64 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -9,7 +9,6 @@ from dvc.exceptions import DvcException, HTTPError from dvc.progress import Tqdm from dvc.remote.base import RemoteBASE from dvc.scheme import Schemes -from dvc.utils.compat import open logger = logging.getLogger(__name__) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index a25a986b5..a454b800b 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -29,7 +29,6 @@ from dvc.utils import relpath from dvc.utils import tmp_fname from dvc.utils import walk_files from dvc.utils.compat import fspath_py35 -from dvc.utils.compat import open from dvc.utils.fs import move from dvc.utils.fs import remove diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index 37dde23da..7f9a59bcb 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -7,6 +7,7 @@ import os import threading from concurrent.futures import ThreadPoolExecutor from contextlib import closing, contextmanager +from urllib.parse import urlparse from funcy import memoize, wrap_with @@ -17,7 +18,6 @@ from dvc.remote.base import RemoteBASE from dvc.remote.pool import get_connection from dvc.scheme import Schemes from dvc.utils import to_chunks -from dvc.utils.compat import urlparse logger = logging.getLogger(__name__) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 8141d3e55..fe59db124 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -5,7 +5,7 @@ from functools import wraps from itertools import chain from dvc.ignore import CleanTree -from dvc.utils.compat import FileNotFoundError, fspath_py35, open as _open +from dvc.utils.compat import fspath_py35 from funcy import cached_property @@ -471,7 +471,7 @@ class Repo(object): cache_file = fspath_py35(cache_file) if os.path.exists(cache_file): - return _open(cache_file, mode=mode, encoding=encoding) + return open(cache_file, mode=mode, encoding=encoding) try: remote_obj = self.cloud.get_remote(remote) @@ -482,7 +482,7 @@ class Repo(object): cache_info = out.get_used_cache(remote=remote) self.cloud.pull(cache_info, remote=remote) - return _open(cache_file, mode=mode, encoding=encoding) + return open(cache_file, mode=mode, encoding=encoding) def close(self): self.scm.close() diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 929d0acfd..e55a34dc3 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -26,7 +26,7 @@ def gc( jobs=None, repos=None, ): - from dvc.utils.compat import ExitStack + from contextlib import ExitStack from dvc.repo import Repo all_repos = [] diff --git a/dvc/repo/get.py b/dvc/repo/get.py index 3c7e1e68b..db591dbb9 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -1,6 +1,5 @@ import logging import os -from dvc.utils.compat import FileNotFoundError import shortuuid diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py index 646b3022f..375d7a38b 100644 --- a/dvc/repo/metrics/show.py +++ b/dvc/repo/metrics/show.py @@ -10,7 +10,6 @@ from jsonpath_ng.ext import parse from dvc.exceptions import NoMetricsError from dvc.exceptions import OutputNotFoundError from dvc.repo import locked -from dvc.utils.compat import open NO_METRICS_FILE_AT_REFERENCE_WARNING = ( "Metrics file '{}' does not exist at the reference '{}'." diff --git a/dvc/rwlock.py b/dvc/rwlock.py index 9da97fe4b..f7afa5f55 100644 --- a/dvc/rwlock.py +++ b/dvc/rwlock.py @@ -9,7 +9,6 @@ from funcy.py3 import lmap, lfilter, lmapcat from .exceptions import DvcException from .lock import LockError -from .utils.compat import FileNotFoundError, JSONDecodeError, str from .utils.fs import relpath INFO_SCHEMA = {"pid": int, "cmd": str} @@ -42,7 +41,7 @@ def _edit_rwlock(lock_dir): lock = SCHEMA(lock) except FileNotFoundError: lock = SCHEMA({}) - except JSONDecodeError as exc: + except json.JSONDecodeError as exc: raise RWLockFileCorruptedError(path, cause=exc) except Invalid as exc: raise RWLockFileFormatError(path, cause=exc) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index b214cb3ef..06521f540 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -17,7 +17,6 @@ from dvc.utils import fix_env from dvc.utils import is_binary from dvc.utils import relpath from dvc.utils.fs import path_isin -from dvc.utils.compat import open logger = logging.getLogger(__name__) diff --git a/dvc/scm/tree.py b/dvc/scm/tree.py index 5e28beae9..519cb4159 100644 --- a/dvc/scm/tree.py +++ b/dvc/scm/tree.py @@ -1,6 +1,6 @@ import os -from dvc.utils.compat import open, fspath +from dvc.utils.compat import fspath class BaseTree(object): diff --git a/dvc/stage.py b/dvc/stage.py index a795a6852..34115f610 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -1,4 +1,4 @@ -from dvc.utils.compat import pathlib +import pathlib import logging import os import re diff --git a/dvc/state.py b/dvc/state.py index f371f63b7..26ff2f70d 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -4,6 +4,7 @@ import logging import os import re import sqlite3 +from urllib.parse import urlunparse, urlencode from dvc.config import Config from dvc.exceptions import DvcException @@ -11,8 +12,6 @@ from dvc.utils import current_timestamp from dvc.utils import relpath from dvc.utils import to_chunks from dvc.utils.compat import fspath_py35 -from dvc.utils.compat import urlencode -from dvc.utils.compat import urlunparse from dvc.utils.fs import get_inode from dvc.utils.fs import get_mtime_and_size from dvc.utils.fs import remove diff --git a/dvc/system.py b/dvc/system.py index 5010508c8..181b1b13d 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -4,7 +4,6 @@ import os import shutil from dvc.utils.compat import fspath -from dvc.utils.compat import open logger = logging.getLogger(__name__) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 4eecb1328..4078de00e 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -15,9 +15,7 @@ import nanotime from ruamel.yaml import YAML from shortuuid import uuid -from dvc.utils.compat import fspath -from dvc.utils.compat import fspath_py35 -from dvc.utils.compat import open +from dvc.utils.compat import fspath, fspath_py35 logger = logging.getLogger(__name__) diff --git a/dvc/utils/collections.py b/dvc/utils/collections.py index 03ee4c81e..1ff72a347 100644 --- a/dvc/utils/collections.py +++ b/dvc/utils/collections.py @@ -1,4 +1,4 @@ -from dvc.utils.compat import Mapping +from collections.abc import Mapping def apply_diff(src, dest): diff --git a/dvc/utils/http.py b/dvc/utils/http.py index f478345d1..c64bfdfda 100644 --- a/dvc/utils/http.py +++ b/dvc/utils/http.py @@ -1,8 +1,6 @@ import io from contextlib import contextmanager -from dvc.utils.compat import FileNotFoundError - @contextmanager def open_url(url, mode="r", encoding=None): diff --git a/dvc/utils/stage.py b/dvc/utils/stage.py index 0758c8b46..6913a5771 100644 --- a/dvc/utils/stage.py +++ b/dvc/utils/stage.py @@ -8,7 +8,6 @@ except ImportError: from yaml import SafeLoader from dvc.exceptions import StageFileCorruptedError -from dvc.utils.compat import open def load_stage_file(path): diff --git a/scripts/innosetup/config_gen.py b/scripts/innosetup/config_gen.py index 35e643684..061b3cb2b 100644 --- a/scripts/innosetup/config_gen.py +++ b/scripts/innosetup/config_gen.py @@ -1,8 +1,9 @@ # This script generates config.ini for setup.iss script -from dvc import __version__ -from dvc.utils.compat import ConfigParser +import configparser -config = ConfigParser.ConfigParser() +from dvc import __version__ + +config = configparser.ConfigParser() config.add_section("Version") config.set("Version", "Version", __version__) diff --git a/tests/basic_env.py b/tests/basic_env.py index 8d1f5ce2c..8a6667c45 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -12,7 +12,6 @@ from git import Repo from git.exc import GitCommandNotFound from dvc.repo import Repo as DvcRepo -from dvc.utils.compat import open from dvc.utils.fs import remove diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 256581ce2..058c498f6 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -44,12 +44,13 @@ from global repo template to creating everything inplace, which: """ import os +import pathlib import pytest from funcy.py3 import lmap, retry from dvc.utils import makedirs -from dvc.utils.compat import pathlib, fspath, fspath_py35 +from dvc.utils.compat import fspath, fspath_py35 __all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"] diff --git a/tests/func/test_add.py b/tests/func/test_add.py index ba6b9def5..fd1853809 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -23,7 +23,7 @@ from dvc.system import System from dvc.utils import file_md5 from dvc.utils import LARGE_DIR_SIZE from dvc.utils import relpath -from dvc.utils.compat import range, fspath +from dvc.utils.compat import fspath from dvc.utils.fs import path_isin from dvc.utils.stage import load_stage_file from tests.basic_env import TestDvc diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 42885acbd..057f947c1 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -2,11 +2,13 @@ import filecmp import getpass import os import posixpath +import pathlib import re import shutil import uuid from subprocess import PIPE from subprocess import Popen +from urllib.parse import urljoin import boto3 import paramiko @@ -28,8 +30,6 @@ from dvc.stage import StageFileDoesNotExistError from dvc.system import System from dvc.utils import file_md5 from dvc.utils import relpath -from dvc.utils.compat import pathlib -from dvc.utils.compat import urljoin from dvc.utils.stage import dump_stage_file from dvc.utils.stage import load_stage_file from tests.basic_env import TestDvc diff --git a/tests/unit/test_path_info.py b/tests/unit/test_path_info.py index c9fabf02c..99476ff95 100644 --- a/tests/unit/test_path_info.py +++ b/tests/unit/test_path_info.py @@ -1,11 +1,11 @@ import copy +import pathlib import pytest from dvc.path_info import CloudURLInfo from dvc.path_info import PathInfo from dvc.path_info import URLInfo -from dvc.utils.compat import pathlib TEST_DEPTH = len(pathlib.Path(__file__).parents) + 1 diff --git a/tests/utils/httpd.py b/tests/utils/httpd.py index 92027ae75..2a3091eb3 100644 --- a/tests/utils/httpd.py +++ b/tests/utils/httpd.py @@ -1,14 +1,7 @@ import hashlib import os import threading - -# Moved in Python 2 -> 3, used only here. -# Not adding to dvc.utils.compat to not load http.server for non-test runs. -try: - from http.server import HTTPServer -except ImportError: - from BaseHTTPServer import HTTPServer - +from http.server import HTTPServer from RangeHTTPServer import RangeRequestHandler