py3: get rid of is_py checks

This commit is contained in:
Mr. Outis
2019-12-30 18:53:36 -06:00
parent 21808112df
commit 886afdb203
7 changed files with 62 additions and 131 deletions
+2 -5
View File
@@ -14,11 +14,8 @@ def fix_subparsers(subparsers):
Args:
subparsers: subparsers to fix.
"""
from dvc.utils.compat import is_py3
if is_py3: # pragma: no cover
subparsers.required = True
subparsers.dest = "cmd"
subparsers.required = True
subparsers.dest = "cmd"
def append_doc_link(help_message, path):
+1 -4
View File
@@ -1,7 +1,6 @@
"""Use heuristics to guess if it is a text file or a binary file."""
from __future__ import unicode_literals
from dvc.utils.compat import is_py3
from dvc.utils.compat import open
# Based on https://eli.thegreenplace.net/2011/10/19/
@@ -13,9 +12,7 @@ from dvc.utils.compat import open
# in py2.
#
def _int2byte(i):
if is_py3:
return bytes((i,))
return chr(i)
return bytes((i,))
TEXT_CHARS = b"".join(_int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b"
+53 -66
View File
@@ -7,10 +7,10 @@ import time
from datetime import timedelta
import zc.lockfile
import flufl.lock
from dvc.exceptions import DvcException
from dvc.utils import format_link
from dvc.utils.compat import is_py3, is_py2
from dvc.progress import Tqdm
DEFAULT_TIMEOUT = 5
@@ -76,83 +76,70 @@ class Lock(object):
self.unlock()
if is_py3:
import flufl.lock
class HardlinkLock(flufl.lock.Lock):
"""Class for dvc repo lock.
class HardlinkLock(flufl.lock.Lock):
"""Class for dvc repo lock.
Args:
lockfile (str): the lock filename
in.
tmp_dir (str): a directory to store claim files.
"""
Args:
lockfile (str): the lock filename
in.
tmp_dir (str): a directory to store claim files.
"""
def __init__(self, lockfile, tmp_dir=None, **kwargs):
import socket
def __init__(self, lockfile, tmp_dir=None, **kwargs):
import socket
self._tmp_dir = tmp_dir
self._tmp_dir = tmp_dir
# NOTE: this is basically Lock.__init__ copy-paste, except that
# instead of using `socket.getfqdn()` we use `socket.gethostname()`
# to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
# anything, which is way too slow. `gethostname()` is actually a
# fallback for `getfqdn()` when it is not able to resolve a
# canonical hostname through network. The claimfile that uses
# `self._hostname` is still usable, as it uses `pid` and random
# number to generate the resulting lock file name, which is unique
# enough for our application.
#
# [1] https://github.com/iterative/dvc/issues/2582
self._hostname = socket.gethostname()
# NOTE: this is basically Lock.__init__ copy-paste, except that
# instead of using `socket.getfqdn()` we use `socket.gethostname()`
# to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
# anything, which is way too slow. `gethostname()` is actually a
# fallback for `getfqdn()` when it is not able to resolve a
# canonical hostname through network. The claimfile that uses
# `self._hostname` is still usable, as it uses `pid` and random
# number to generate the resulting lock file name, which is unique
# enough for our application.
#
# [1] https://github.com/iterative/dvc/issues/2582
self._hostname = socket.gethostname()
self._lockfile = lockfile
self._lifetime = timedelta(days=365) # Lock for good by default
self._separator = flufl.lock.SEP
self._set_claimfile()
self._owned = True
self._retry_errnos = []
self._lockfile = lockfile
self._lifetime = timedelta(days=365) # Lock for good by default
self._separator = flufl.lock.SEP
self._set_claimfile()
self._owned = True
self._retry_errnos = []
@property
def lockfile(self):
return self._lockfile
@property
def lockfile(self):
return self._lockfile
def lock(self):
try:
super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
except flufl.lock.TimeOutError:
raise LockError(FAILED_TO_LOCK_MESSAGE)
def lock(self):
try:
super(Lock, self).lock(timedelta(seconds=DEFAULT_TIMEOUT))
except flufl.lock.TimeOutError:
raise LockError(FAILED_TO_LOCK_MESSAGE)
def _set_claimfile(self, pid=None):
super(Lock, self)._set_claimfile(pid)
def _set_claimfile(self, pid=None):
super(Lock, self)._set_claimfile(pid)
if self._tmp_dir is not None:
# Under Windows file path length is limited so we hash it
filename = hashlib.md5(self._claimfile.encode()).hexdigest()
self._claimfile = os.path.join(self._tmp_dir, filename + ".lock")
if self._tmp_dir is not None:
# Under Windows file path length is limited so we hash it
filename = hashlib.md5(self._claimfile.encode()).hexdigest()
self._claimfile = os.path.join(
self._tmp_dir, filename + ".lock"
)
# Fix for __del__ bug in flufl.lock [1] which is causing errors on
# Python shutdown [2].
# [1] https://gitlab.com/warsaw/flufl.lock/issues/7
# [2] https://github.com/iterative/dvc/issues/2573
def __del__(self):
try:
if self._owned:
self.finalize()
except ImportError:
pass
# Fix for __del__ bug in flufl.lock [1] which is causing errors on
# Python shutdown [2].
# [1] https://gitlab.com/warsaw/flufl.lock/issues/7
# [2] https://github.com/iterative/dvc/issues/2573
def __del__(self):
try:
if self._owned:
self.finalize()
except ImportError:
pass
def make_lock(lockfile, tmp_dir=None, friendly=False, hardlink_lock=False):
if hardlink_lock and is_py2:
raise DvcException(
"Hardlink locks are not supported on Python <3.5. "
"See `hardlink_lock` in {}".format(
format_link("man.dvc.org/config#core")
)
)
cls = HardlinkLock if hardlink_lock else Lock
return cls(lockfile, tmp_dir=tmp_dir, friendly=friendly)
+1 -7
View File
@@ -12,7 +12,6 @@ from dvc.exceptions import NotDvcRepoError
from dvc.external_repo import clean_repos
from dvc.logger import FOOTER
from dvc.remote.pool import close_pools
from dvc.utils.compat import is_py2
# Workaround for CPython bug. See [1] and [2] for more info.
@@ -60,12 +59,7 @@ def main(argv=None):
except DvcParserError:
ret = 254
except Exception as exc: # pylint: disable=broad-except
if isinstance(exc, UnicodeError) and is_py2:
logger.exception(
"unicode is not supported in DVC for Python 2 "
"(end-of-life January 1, 2020), please upgrade to Python 3"
)
elif isinstance(exc, OSError) and exc.errno == errno.EMFILE:
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
logger.exception(
"too many open files, please increase your `ulimit`"
)
-37
View File
@@ -3,26 +3,17 @@ from __future__ import unicode_literals
import os
import posixpath
import sys
from funcy import cached_property
from dvc.utils import relpath
from dvc.utils.compat import basestring
from dvc.utils.compat import builtin_str
from dvc.utils.compat import is_py2
from dvc.utils.compat import pathlib
from dvc.utils.compat import str
from dvc.utils.compat import urlparse
# On Python 2.7/Windows sys.getfilesystemencoding() is set to mbcs,
# which is lossy, thus we can't use that,
# see https://github.com/mcmtroffaes/pathlib2/issues/56.
if is_py2:
fs_encoding = "utf-8"
class _BasePath(object):
def overlaps(self, other):
if isinstance(other, basestring):
@@ -91,34 +82,6 @@ class PathInfo(pathlib.PurePath, _BasePath):
n = len(other._cparts)
return len(self._cparts) > n and self._cparts[:n] == other._cparts
# pathlib2 uses bytes internally in Python 2, and we use unicode everywhere
# for paths in both pythons, thus we need this glue.
if is_py2:
__unicode__ = __str__
def __str__(self):
return self.__unicode__().encode(sys.getfilesystemencoding())
@classmethod
def _parse_args(cls, args):
args = [
a.encode(fs_encoding)
if isinstance(a, unicode) # noqa: F821
else a
for a in args
]
return super(PathInfo, cls)._parse_args(args)
@property
def name(self):
return super(PathInfo, self).name.decode(fs_encoding)
def __fspath__(self): # noqa: F811
return pathlib.PurePath.__str__(self).decode(fs_encoding)
def with_name(self, name):
return pathlib.PurePath.with_name(self, name.encode(fs_encoding))
class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
pass
+1 -2
View File
@@ -12,7 +12,6 @@ from dvc.utils import current_timestamp
from dvc.utils import relpath
from dvc.utils import to_chunks
from dvc.utils.compat import fspath_py35
from dvc.utils.compat import is_py2
from dvc.utils.compat import urlencode
from dvc.utils.compat import urlunparse
from dvc.utils.fs import get_inode
@@ -486,7 +485,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
def _connect_sqlite(filename, options):
# Connect by URI was added in Python 3.4 and sqlite 3.7.7,
# we ignore options, which should be fine unless repo is on old NFS/CIFS
if is_py2 or sqlite3.sqlite_version_info < (3, 7, 7):
if sqlite3.sqlite_version_info < (3, 7, 7):
return sqlite3.connect(filename)
uri = _build_sqlite_uri(filename, options)
+4 -10
View File
@@ -19,7 +19,6 @@ from dvc.stage import StageFileBadNameError
from dvc.stage import StageFileDoesNotExistError
from dvc.system import System
from dvc.utils import relpath, walk_files
from dvc.utils.compat import is_py2
from dvc.utils.stage import dump_stage_file
from dvc.utils.stage import load_stage_file
from tests.basic_env import TestDvc
@@ -435,16 +434,11 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
ret = main(["add", self.DATA_DIR])
self.assertEqual(ret, 0)
if os.name == "nt" and is_py2:
from jaraco.windows.filesystem import readlink
else:
readlink = os.readlink
self.assertTrue(System.is_symlink(self.FOO))
old_foo_link = readlink(self.FOO)
old_foo_link = os.readlink(self.FOO)
self.assertTrue(System.is_symlink(self.DATA))
old_data_link = readlink(self.DATA)
old_data_link = os.readlink(self.DATA)
old_cache_dir = self.dvc.cache.local.cache_dir
new_cache_dir = old_cache_dir + "_new"
@@ -457,10 +451,10 @@ class TestCheckoutMovedCacheDirWithSymlinks(TestDvc):
self.assertEqual(ret, 0)
self.assertTrue(System.is_symlink(self.FOO))
new_foo_link = readlink(self.FOO)
new_foo_link = os.readlink(self.FOO)
self.assertTrue(System.is_symlink(self.DATA))
new_data_link = readlink(self.DATA)
new_data_link = os.readlink(self.DATA)
self.assertEqual(
relpath(old_foo_link, old_cache_dir),